From 483eb2f56657e8e7f419ab1a4fab8dce9ade8609 Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Sat, 27 Apr 2024 20:24:20 +0200 Subject: Adding upstream version 14.2.21. Signed-off-by: Daniel Baumann --- src/spdk/intel-ipsec-mb/include/datastruct.asm | 235 ++++++++++++++ src/spdk/intel-ipsec-mb/include/dbgprint.asm | 409 +++++++++++++++++++++++++ src/spdk/intel-ipsec-mb/include/des_utils.h | 134 ++++++++ src/spdk/intel-ipsec-mb/include/memcpy.asm | 344 +++++++++++++++++++++ src/spdk/intel-ipsec-mb/include/os.asm | 58 ++++ src/spdk/intel-ipsec-mb/include/reg_sizes.asm | 115 +++++++ src/spdk/intel-ipsec-mb/include/save_xmms.asm | 102 ++++++ src/spdk/intel-ipsec-mb/include/save_xmms.h | 37 +++ 8 files changed, 1434 insertions(+) create mode 100644 src/spdk/intel-ipsec-mb/include/datastruct.asm create mode 100644 src/spdk/intel-ipsec-mb/include/dbgprint.asm create mode 100644 src/spdk/intel-ipsec-mb/include/des_utils.h create mode 100644 src/spdk/intel-ipsec-mb/include/memcpy.asm create mode 100644 src/spdk/intel-ipsec-mb/include/os.asm create mode 100644 src/spdk/intel-ipsec-mb/include/reg_sizes.asm create mode 100644 src/spdk/intel-ipsec-mb/include/save_xmms.asm create mode 100644 src/spdk/intel-ipsec-mb/include/save_xmms.h (limited to 'src/spdk/intel-ipsec-mb/include') diff --git a/src/spdk/intel-ipsec-mb/include/datastruct.asm b/src/spdk/intel-ipsec-mb/include/datastruct.asm new file mode 100644 index 00000000..0ab1113a --- /dev/null +++ b/src/spdk/intel-ipsec-mb/include/datastruct.asm @@ -0,0 +1,235 @@ +;; +;; Copyright (c) 2012-2018, Intel Corporation +;; +;; Redistribution and use in source and binary forms, with or without +;; modification, are permitted provided that the following conditions are met: +;; +;; * Redistributions of source code must retain the above copyright notice, +;; this list of conditions and the following disclaimer. +;; * Redistributions in binary form must reproduce the above copyright +;; notice, this list of conditions and the following disclaimer in the +;; documentation and/or other materials provided with the distribution. +;; * Neither the name of Intel Corporation nor the names of its contributors +;; may be used to endorse or promote products derived from this software +;; without specific prior written permission. +;; +;; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +;; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +;; IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +;; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE +;; FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +;; DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +;; SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +;; CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +;; OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +;; + +; Macros for defining data structures + +; Usage example + +;START_FIELDS ; JOB_AES +;;; name size align +;FIELD _plaintext, 8, 8 ; pointer to plaintext +;FIELD _ciphertext, 8, 8 ; pointer to ciphertext +;FIELD _IV, 16, 8 ; IV +;FIELD _keys, 8, 8 ; pointer to keys +;FIELD _len, 4, 4 ; length in bytes +;FIELD _status, 4, 4 ; status enumeration +;FIELD _user_data, 8, 8 ; pointer to user data +;UNION _union, size1, align1, \ +; size2, align2, \ +; size3, align3, \ +; ... +;END_FIELDS +;%assign _JOB_AES_size _FIELD_OFFSET +;%assign _JOB_AES_align _STRUCT_ALIGN + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +; Alternate "struc-like" syntax: +; STRUCT job_aes2 +; RES_Q .plaintext, 1 +; RES_Q .ciphertext, 1 +; RES_DQ .IV, 1 +; RES_B .nested, _JOB_AES_SIZE, _JOB_AES_ALIGN +; RES_U .union, size1, align1, \ +; size2, align2, \ +; ... +; ENDSTRUCT +; ; Following only needed if nesting +; %assign job_aes2_size _FIELD_OFFSET +; %assign job_aes2_align _STRUCT_ALIGN +; +; RES_* macros take a name, a count and an optional alignment. +; The count in in terms of the base size of the macro, and the +; default alignment is the base size. +; The macros are: +; Macro Base size +; RES_B 1 +; RES_W 2 +; RES_D 4 +; RES_Q 8 +; RES_DQ 16 +; RES_Y 32 +; RES_Z 64 +; +; RES_U defines a union. It's arguments are a name and two or more +; pairs of "size, alignment" +; +; The two assigns are only needed if this structure is being nested +; within another. Even if the assigns are not done, one can still use +; STRUCT_NAME_size as the size of the structure. +; +; Note that for nesting, you still need to assign to STRUCT_NAME_size. +; +; The differences between this and using "struc" directly are that each +; type is implicitly aligned to its natural length (although this can be +; over-ridden with an explicit third parameter), and that the structure +; is padded at the end to its overall alignment. +; + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +%ifndef _DATASTRUCT_ASM_ +%define _DATASTRUCT_ASM_ + +;; START_FIELDS +%macro START_FIELDS 0 +%assign _FIELD_OFFSET 0 +%assign _STRUCT_ALIGN 0 +%endm + +;; FIELD name size align +%macro FIELD 3 +%define %%name %1 +%define %%size %2 +%define %%align %3 + +%assign _FIELD_OFFSET (_FIELD_OFFSET + (%%align) - 1) & (~ ((%%align)-1)) +%%name equ _FIELD_OFFSET +%assign _FIELD_OFFSET _FIELD_OFFSET + (%%size) +%if (%%align > _STRUCT_ALIGN) +%assign _STRUCT_ALIGN %%align +%endif +%endm + +;; END_FIELDS +%macro END_FIELDS 0 +%assign _FIELD_OFFSET (_FIELD_OFFSET + _STRUCT_ALIGN-1) & (~ (_STRUCT_ALIGN-1)) +%endm + +%macro UNION 5-* +%if (0 == (%0 & 1)) + %error EVEN number of parameters to UNION Macro + %err +%endif +%rotate 1 + %assign _UNION_SIZE %1 + %assign _UNION_ALIGN %2 +%rep (%0 - 3)/2 + %rotate 2 + %if (%1 > _UNION_SIZE) + %assign _UNION_SIZE %1 + %endif + %if (%2 > _UNION_ALIGN) + %assign _UNION_ALIGN %2 + %endif +%endrep +%rotate 2 +FIELD %1, _UNION_SIZE, _UNION_ALIGN +%endm + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +%macro STRUCT 1 +START_FIELDS +struc %1 +%endm + +%macro ENDSTRUCT 0 +%assign %%tmp _FIELD_OFFSET +END_FIELDS +%assign %%tmp (_FIELD_OFFSET - %%tmp) +%if (%%tmp > 0) + resb %%tmp +%endif +endstruc +%endm + +;; RES_int name size align +%macro RES_int 3 +%define %%name %1 +%define %%size %2 +%define %%align %3 + +%assign _FIELD_OFFSET (_FIELD_OFFSET + (%%align) - 1) & (~ ((%%align)-1)) +align %%align +%%name resb %%size +%assign _FIELD_OFFSET _FIELD_OFFSET + (%%size) +%if (%%align > _STRUCT_ALIGN) +%assign _STRUCT_ALIGN %%align +%endif +%endm + + + +; macro RES_B name, size [, align] +%macro RES_B 2-3 1 +RES_int %1, %2, %3 +%endm + +; macro RES_W name, size [, align] +%macro RES_W 2-3 2 +RES_int %1, 2*(%2), %3 +%endm + +; macro RES_D name, size [, align] +%macro RES_D 2-3 4 +RES_int %1, 4*(%2), %3 +%endm + +; macro RES_Q name, size [, align] +%macro RES_Q 2-3 8 +RES_int %1, 8*(%2), %3 +%endm + +; macro RES_DQ name, size [, align] +%macro RES_DQ 2-3 16 +RES_int %1, 16*(%2), %3 +%endm + +; macro RES_Y name, size [, align] +%macro RES_Y 2-3 32 +RES_int %1, 32*(%2), %3 +%endm + +; macro RES_Z name, size [, align] +%macro RES_Z 2-3 64 +RES_int %1, 64*(%2), %3 +%endm + + +%macro RES_U 5-* +%if (0 == (%0 & 1)) + %error EVEN number of parameters to RES_U Macro + %err +%endif +%rotate 1 + %assign _UNION_SIZE %1 + %assign _UNION_ALIGN %2 +%rep (%0 - 3)/2 + %rotate 2 + %if (%1 > _UNION_SIZE) + %assign _UNION_SIZE %1 + %endif + %if (%2 > _UNION_ALIGN) + %assign _UNION_ALIGN %2 + %endif +%endrep +%rotate 2 +RES_int %1, _UNION_SIZE, _UNION_ALIGN +%endm + +%endif ; end ifdef _DATASTRUCT_ASM_ diff --git a/src/spdk/intel-ipsec-mb/include/dbgprint.asm b/src/spdk/intel-ipsec-mb/include/dbgprint.asm new file mode 100644 index 00000000..8da9c7b2 --- /dev/null +++ b/src/spdk/intel-ipsec-mb/include/dbgprint.asm @@ -0,0 +1,409 @@ +;; +;; Copyright (c) 2012-2018, Intel Corporation +;; +;; Redistribution and use in source and binary forms, with or without +;; modification, are permitted provided that the following conditions are met: +;; +;; * Redistributions of source code must retain the above copyright notice, +;; this list of conditions and the following disclaimer. +;; * Redistributions in binary form must reproduce the above copyright +;; notice, this list of conditions and the following disclaimer in the +;; documentation and/or other materials provided with the distribution. +;; * Neither the name of Intel Corporation nor the names of its contributors +;; may be used to endorse or promote products derived from this software +;; without specific prior written permission. +;; +;; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +;; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +;; IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +;; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE +;; FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +;; DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +;; SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +;; CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +;; OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +;; + +; Macros for "printing" for debug purposes from within asm code +; +; The basic macros are: +; DBGPRINT16, DBGPRINT32, DBGPRINT64, DBGPRINT_XMM, DBGPRINT_YMM, DBGPRINT_ZMM +; These are called with 1 or more arguments, all of which are of the +; size/type as specified in the name. E.g. +; DBGPRINT64 reg1, reg2, reg3, ... +; +; There is also a macro DEBUGPRINTL that takes one argument, a string. E.g. +; DBGPRINTL "hit this point in the code" +; +; There are also variations on these with the "DBGPRINT" suffixed with "L", e.g. +; DBGPRINTL64. These take two or more arguments, where the first is a string, +; and the rest are of the specified type, e.g. +; DBGPRINTL64 "Rindex", Rindex +; Essentially, this is the same as a DBGPRINTL followed by DBGPRINT64. +; +; If DO_DBGPRINT is defined, then the macros write the debug information into +; a buffer. If DO_DBGPRINT is *not* defined, then the macros expand to nothing. +; +; CAVEAT: The macros need a GPR. Currently, it uses R15. If the first register +; argument is R15, then it will use R14. This means that if you try +; DBGPRINTL64 "text", rax, r15 +; you will not get the proper value of r15. +; One way to avoid this issue is to not use multiple registers on the same line +; if the register types are GPR (i.e. this is not an issue for printing XMM +; registers). E.g the above could be done with: +; DBGPRINTL64 "test", rax +; DBGPRINT64 r15 +; +; Note also that the macros only check for r15. Thus is you tried something +; like (after token expansion): +; DBGPRINT32 r15d +; you won't get the right results. If you want to display r15d, you should +; print it as the 64-bit r15. +; +; To actually print the data, from your C code include the file +; "dbgprint.h". The default buffer size is 16kB. If you want to change +; that, #define DBG_BUFFER_SIZE before including "dbgprint.h". +; +; Then, (after your asm routine(s) have returned, call +; print_debug() or print_debug(file pointer) +; If you do not specify a file pointer, it defaults to stdout. +; +; Printing the debug data also resets the write pointer to the beginning, +; effectively "deleting" the previous messages. +; + +;%define DO_DBGPRINT +%ifdef DO_DBGPRINT +extern pDebugBuffer +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; DBGPRINT_INT size, param, ... +%macro DBGPRINT_INT 2-* +%ifidni %2,r15 +%xdefine %%reg r14 +%else +%xdefine %%reg r15 +%endif +%xdefine %%size %1 +%rotate 1 + push %%reg + mov %%reg, [pDebugBuffer] +%rep %0 - 1 + mov byte [%%reg], %%size + %if (%%size == 2) + mov word [%%reg+1], %1 + %elif (%%size == 4) + mov dword [%%reg+1], %1 + %elif (%%size == 8) + mov qword [%%reg+1], %1 + %elif (%%size == 16) + movdqu oword [%%reg+1], %1 + %elif (%%size == 32) + vmovdqu [%%reg+1], %1 + %elif (%%size == 64) + vmovdqu32 [%%reg+1], %1 + %else + %error invalid size %%size + %endif + add %%reg, %%size+1 +%rotate 1 +%endrep + mov [pDebugBuffer], %%reg + pop %%reg +%endmacro +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; DBGPRINTL_INT size, label, param, ... +%macro DBGPRINTL_INT 3-* +%ifidni %3,r15 +%xdefine %%reg r14 +%else +%xdefine %%reg r15 +%endif +%xdefine %%size %1 +%rotate 1 + push %%reg + mov %%reg, [pDebugBuffer] + + mov byte [%%reg], 0x57 +section .data +%%lab: db %1, 0 +section .text + mov qword [%%reg+1], %%lab + add %%reg, 8+1 +%rotate 1 + +%rep %0 - 2 + mov byte [%%reg], %%size +%if (%%size == 2) + mov word [%%reg+1], %1 +%elif (%%size == 4) + mov dword [%%reg+1], %1 +%elif (%%size == 8) + mov qword [%%reg+1], %1 +%elif (%%size == 16) + movdqu oword [%%reg+1], %1 +%elif (%%size == 32) + vmovdqu [%%reg+1], %1 +%elif (%%size == 64) + vmovdqu32 [%%reg+1], %1 +%else +%error invalid size %%size +%endif + add %%reg, %%size+1 +%rotate 1 +%endrep + mov [pDebugBuffer], %%reg + pop %%reg +%endmacro +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; DBGPRINTL* data, ... +%macro DBGPRINT16 1+ + DBGPRINT_INT 2, %1 +%endmacro +%macro DBGPRINT32 1+ + DBGPRINT_INT 4, %1 +%endmacro +%macro DBGPRINT64 1+ + DBGPRINT_INT 8, %1 +%endmacro +%macro DBGPRINT_XMM 1+ + DBGPRINT_INT 16, %1 +%endmacro +%macro DBGPRINT_YMM 1+ + DBGPRINT_INT 32, %1 +%endmacro +%macro DBGPRINT_ZMM 1+ + DBGPRINT_INT 64, %1 +%endmacro +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; DBGPRINTL* label, data, ... +%macro DBGPRINTL16 2+ + DBGPRINTL_INT 2, %1, %2 +%endmacro +%macro DBGPRINTL32 2+ + DBGPRINTL_INT 4, %1, %2 +%endmacro +%macro DBGPRINTL64 2+ + DBGPRINTL_INT 8, %1, %2 +%endmacro +%macro DBGPRINTL_XMM 2+ + DBGPRINTL_INT 16, %1, %2 +%endmacro +%macro DBGPRINTL_YMM 2+ + DBGPRINTL_INT 32, %1, %2 +%endmacro +%macro DBGPRINTL_ZMM 2+ + DBGPRINTL_INT 64, %1, %2 +%endmacro +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +%macro DBGPRINTL 1 + push r15 + mov r15, [pDebugBuffer] + + mov byte [r15], 0x57 +section .data +%%lab: db %1, 0 +section .text + mov qword [r15+1], %%lab + add r15, 8+1 + + mov [pDebugBuffer], r15 + pop r15 +%endmacro +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +%else +%macro DBGPRINT16 1+ +%endmacro +%macro DBGPRINT32 1+ +%endmacro +%macro DBGPRINT64 1+ +%endmacro +%macro DBGPRINT_XMM 1+ +%endmacro +%macro DBGPRINT_YMM 1+ +%endmacro +%macro DBGPRINT_ZMM 1+ +%endmacro +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +%macro DBGPRINTL16 2+ +%endmacro +%macro DBGPRINTL32 2+ +%endmacro +%macro DBGPRINTL64 2+ +%endmacro +%macro DBGPRINTL_XMM 2+ +%endmacro +%macro DBGPRINTL_YMM 2+ +%endmacro +%macro DBGPRINTL_ZMM 2+ +%endmacro +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +%macro DBGPRINTL 1 +%endmacro +%endif + + + +%if 0 ; OLD +%macro DBGPRINTL_ZMM 2-* + push rax + mov rax, [pDebugBuffer] + + mov byte [rax], 0x57 +section .data +%%lab: db %1, 0 +section .text + mov qword [rax+1], %%lab + add rax, 8+1 +%rotate 1 + +%rep %0 - 1 + mov byte [rax], 64 + vmovdqu32 [rax+1], %1 +%rotate 1 + add rax, 64+1 +%endrep + mov [pDebugBuffer], rax + pop rax +%endmacro +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +%macro DBGPRINT_ZMM 1-* + push rax + mov rax, [pDebugBuffer] +%rep %0 + mov byte [rax], 64 + vmovdqu32 [rax+1], %1 +%rotate 1 + add rax, 64+1 +%endrep + mov [pDebugBuffer], rax + pop rax +%endmacro +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +%macro DBGPRINT_YMM 1-* + push rax + mov rax, [pDebugBuffer] +%rep %0 + mov byte [rax], 32 + vmovdqu [rax+1], %1 +%rotate 1 + add rax, 32+1 +%endrep + mov [pDebugBuffer], rax + pop rax +%endmacro +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +%macro DBGPRINT_XMM 1-* + push rax + mov rax, [pDebugBuffer] +%rep %0 + mov byte [rax], 16 + vmovdqu oword [rax+1], %1 +%rotate 1 + add rax, 16+1 +%endrep + mov [pDebugBuffer], rax + pop rax +%endmacro +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +%macro DBGPRINTL64 2-* + push rax + mov rax, [pDebugBuffer] + + mov byte [rax], 0x57 +section .data +%%lab: db %1, 0 +section .text + mov qword [rax+1], %%lab + add rax, 8+1 +%rotate 1 + +%rep %0 - 1 + mov byte [rax], 8 + mov qword [rax+1], %1 +%rotate 1 + add rax, 8+1 +%endrep + mov [pDebugBuffer], rax + pop rax +%endmacro +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +%macro DBGPRINT64 1-* + push rax + mov rax, [pDebugBuffer] +%rep %0 + mov byte [rax], 8 + mov qword [rax+1], %1 +%rotate 1 + add rax, 8+1 +%endrep + mov [pDebugBuffer], rax + pop rax +%endmacro +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +%macro DBGPRINT32 1-* + push rax + mov rax, [pDebugBuffer] +%rep %0 + mov byte [rax], 4 + mov dword [rax+1], %1 +%rotate 1 + add rax, 4+1 +%endrep + mov [pDebugBuffer], rax + pop rax +%endmacro +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +%macro DBGPRINT16 1-* + push rax + mov rax, [pDebugBuffer] +%rep %0 + mov byte [rax], 2 + mov word [rax+1], %1 +%rotate 1 + add rax, 2+1 +%endrep + mov [pDebugBuffer], rax + pop rax +%endmacro +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +%macro DBGPRINT_LAB 1 + push rax + mov rax, [pDebugBuffer] + + mov byte [rax], 0x57 +section .data +%%lab: db %1, 0 +section .text + mov qword [rax+1], %%lab + add rax, 8+1 + + mov [pDebugBuffer], rax + pop rax +%endmacro +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +%macro DBGHIST 2 + inc dword [%1 + 4 * %2] +%endmacro +%macro DBGPRINT_ZMM 1-* +%endmacro +%macro DBGPRINT_YMM 1-* +%endmacro +%macro DBGPRINT_XMM 1-* +%endmacro +%macro DBGPRINT64 1-* +%endmacro +%macro DBGPRINT32 1-* +%endmacro +%macro DBGPRINT16 1-* +%endmacro +%macro DBGHIST 2 +%endmacro +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +%endif ; ifdef 0 ; OLD diff --git a/src/spdk/intel-ipsec-mb/include/des_utils.h b/src/spdk/intel-ipsec-mb/include/des_utils.h new file mode 100644 index 00000000..4358132d --- /dev/null +++ b/src/spdk/intel-ipsec-mb/include/des_utils.h @@ -0,0 +1,134 @@ +/******************************************************************************* + Copyright (c) 2017-2018, Intel Corporation + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of Intel Corporation nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE + FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*******************************************************************************/ + +/* DES utility functions and macros */ + +#ifndef DES_UTILS_H +#define DES_UTILS_H + +#include +#include "intel-ipsec-mb.h" + +/** + * @brief Gets selected bit value out of a 64-bit word + * + * @param val 64-bit word + * @param n bit number (0 to 63) to get value of + * + * @return n-th bit value (0 or 1 value only) + */ +__forceinline +uint64_t bit_get64b(const uint64_t val, const unsigned n) +{ + IMB_ASSERT(n < 64); + return (val >> n) & UINT64_C(1); +} + +/** + * @brief Sets selected bit in a 64-bit word + * + * @param val 64-bit word + * @param n bit number (0 to 63) to get value of + * @param b bit value (0 or 1) + * + * @return val with n-th bit set to value b + */ +__forceinline +uint64_t bit_set64b(const uint64_t val, const unsigned n, const uint64_t b) +{ + const uint64_t m = UINT64_C(1) << n; + + IMB_ASSERT(n < 64); + return (val & (~m)) | (b << n); +} + +/** + * @brief Permutes bits in a 64-bit word as described by pattern + * + * The function goes through pattern array from index 0 to 'size' (max 63). + * It sets output bit number 'index' to value of + * bit number 'pattern[index] - 1' from 'in'. + * + * @param in 64-bit word to be permuted + * @param pattern pointer to array defining the permutation + * @param size is size of the permutation pattern + * + * @return permuted in word as described by the pattern + */ +__forceinline +uint64_t permute_64b(const uint64_t in, const uint8_t *pattern, const int size) +{ + uint64_t out = 0; + int n = 0; + + IMB_ASSERT(size <= 64); + + for (n = 0; n < size; n++) { + /* '-1' is required as bit numbers in FIPS start with 1 not 0 */ + const int m = ((int) pattern[n]) - 1; + const uint64_t bit_val = bit_get64b(in, m); + + out = bit_set64b(out, n, bit_val); + } + + return out; +} + +static const uint8_t reflect_tab[16] = { + /* [ 0] 0000 => 0000 */ 0, /* [ 1] 0001 => 1000 */ 8, + /* [ 2] 0010 => 0100 */ 4, /* [ 3] 0011 => 1100 */ 12, + /* [ 4] 0100 => 0010 */ 2, /* [ 5] 0101 => 1010 */ 10, + /* [ 6] 0110 => 0110 */ 6, /* [ 7] 0111 => 1110 */ 14, + /* [ 8] 1000 => 0001 */ 1, /* [ 9] 1001 => 1001 */ 9, + /* [10] 1010 => 0101 */ 5, /* [11] 1011 => 1101 */ 13, + /* [12] 1100 => 0011 */ 3, /* [13] 1101 => 1011 */ 11, + /* [14] 1110 => 0111 */ 7, /* [15] 1111 => 1111 */ 15 +}; + +__forceinline +uint8_t reflect_8b(const uint8_t pb) +{ + return reflect_tab[pb >> 4] | (reflect_tab[pb & 15] << 4); +} + +__forceinline +uint64_t load64_reflect(const void *key) +{ + const uint8_t *kb = (const uint8_t *) key; + + return ((uint64_t) reflect_8b(kb[0])) | + ((uint64_t) reflect_8b(kb[1])) << 8 | + ((uint64_t) reflect_8b(kb[2])) << 16 | + ((uint64_t) reflect_8b(kb[3])) << 24 | + ((uint64_t) reflect_8b(kb[4])) << 32 | + ((uint64_t) reflect_8b(kb[5])) << 40 | + ((uint64_t) reflect_8b(kb[6])) << 48 | + ((uint64_t) reflect_8b(kb[7])) << 56; +} + + +#endif /* DES_UTILS_H */ diff --git a/src/spdk/intel-ipsec-mb/include/memcpy.asm b/src/spdk/intel-ipsec-mb/include/memcpy.asm new file mode 100644 index 00000000..0844ea8a --- /dev/null +++ b/src/spdk/intel-ipsec-mb/include/memcpy.asm @@ -0,0 +1,344 @@ +;; +;; Copyright (c) 2012-2018, Intel Corporation +;; +;; Redistribution and use in source and binary forms, with or without +;; modification, are permitted provided that the following conditions are met: +;; +;; * Redistributions of source code must retain the above copyright notice, +;; this list of conditions and the following disclaimer. +;; * Redistributions in binary form must reproduce the above copyright +;; notice, this list of conditions and the following disclaimer in the +;; documentation and/or other materials provided with the distribution. +;; * Neither the name of Intel Corporation nor the names of its contributors +;; may be used to endorse or promote products derived from this software +;; without specific prior written permission. +;; +;; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +;; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +;; IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +;; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE +;; FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +;; DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +;; SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +;; CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +;; OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +;; + +%ifndef __MEMCPY_ASM__ +%define __MEMCPY_ASM__ + +%include "reg_sizes.asm" + + +; This file defines a series of macros to copy small to medium amounts +; of data from memory to memory, where the size is variable but limited. +; +; The macros are all called as: +; memcpy DST, SRC, SIZE, TMP0, TMP1, XTMP0, XTMP1, XTMP2, XTMP3 +; with the parameters defined as: +; DST : register: pointer to dst (not modified) +; SRC : register: pointer to src (not modified) +; SIZE : register: length in bytes (not modified) +; TMP0 : 64-bit temp GPR (clobbered) +; TMP1 : 64-bit temp GPR (clobbered) +; XTMP0 : temp XMM (clobbered) +; XTMP1 : temp XMM (clobbered) +; XTMP2 : temp XMM (clobbered) +; XTMP3 : temp XMM (clobbered) +; +; The name indicates the options. The name is of the form: +; memcpy__ +; where: +; is either "sse" or "avx" or "avx2" +; is either "64" or "128" and defines largest value of SIZE +; is blank or "_1". If "_1" then the min SIZE is 1 (otherwise 0) +; is blank or "_ret". If blank, the code falls through. If "ret" +; it does a "ret" at the end +; +; For the avx2 versions, the temp XMM registers need to be YMM registers +; If the SZ is 64, then only two YMM temps are needed, i.e. it is called as: +; memcpy_avx2_64 DST, SRC, SIZE, TMP0, TMP1, YTMP0, YTMP1 +; memcpy_avx2_128 DST, SRC, SIZE, TMP0, TMP1, YTMP0, YTMP1, YTMP2, YTMP3 +; +; For example: +; memcpy_sse_64 : SSE, 0 <= size < 64, falls through +; memcpy_avx_64_1 : AVX1, 1 <= size < 64, falls through +; memcpy_sse_128_ret : SSE, 0 <= size < 128, ends with ret +; mempcy_avx_128_1_ret : AVX1, 1 <= size < 128, ends with ret +; + +%macro memcpy_sse_64 9 + __memcpy_int %1,%2,%3,%4,%5,%6,%7,%8,%9, 0, 64, 0, 0 +%endm + +%macro memcpy_sse_64_1 9 + __memcpy_int %1,%2,%3,%4,%5,%6,%7,%8,%9, 1, 64, 0, 0 +%endm + +%macro memcpy_sse_128 9 + __memcpy_int %1,%2,%3,%4,%5,%6,%7,%8,%9, 0, 128, 0, 0 +%endm + +%macro memcpy_sse_128_1 9 + __memcpy_int %1,%2,%3,%4,%5,%6,%7,%8,%9, 1, 128, 0, 0 +%endm + +%macro memcpy_sse_64_ret 9 + __memcpy_int %1,%2,%3,%4,%5,%6,%7,%8,%9, 0, 64, 1, 0 +%endm + +%macro memcpy_sse_64_1_ret 9 + __memcpy_int %1,%2,%3,%4,%5,%6,%7,%8,%9, 1, 64, 1, 0 +%endm + +%macro memcpy_sse_128_ret 9 + __memcpy_int %1,%2,%3,%4,%5,%6,%7,%8,%9, 0, 128, 1, 0 +%endm + +%macro memcpy_sse_128_1_ret 9 + __memcpy_int %1,%2,%3,%4,%5,%6,%7,%8,%9, 1, 128, 1, 0 +%endm + + +%macro memcpy_sse_16 5 + __memcpy_int %1,%2,%3,%4,%5,,,,, 0, 16, 0, 0 +%endm + +%macro memcpy_sse_16_1 5 + __memcpy_int %1,%2,%3,%4,%5,,,,, 1, 16, 0, 0 +%endm + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +%macro memcpy_avx_64 9 + __memcpy_int %1,%2,%3,%4,%5,%6,%7,%8,%9, 0, 64, 0, 1 +%endm + +%macro memcpy_avx_64_1 9 + __memcpy_int %1,%2,%3,%4,%5,%6,%7,%8,%9, 1, 64, 0, 1 +%endm + +%macro memcpy_avx_128 9 + __memcpy_int %1,%2,%3,%4,%5,%6,%7,%8,%9, 0, 128, 0, 1 +%endm + +%macro memcpy_avx_128_1 9 + __memcpy_int %1,%2,%3,%4,%5,%6,%7,%8,%9, 1, 128, 0, 1 +%endm + +%macro memcpy_avx_64_ret 9 + __memcpy_int %1,%2,%3,%4,%5,%6,%7,%8,%9, 0, 64, 1, 1 +%endm + +%macro memcpy_avx_64_1_ret 9 + __memcpy_int %1,%2,%3,%4,%5,%6,%7,%8,%9, 1, 64, 1, 1 +%endm + +%macro memcpy_avx_128_ret 9 + __memcpy_int %1,%2,%3,%4,%5,%6,%7,%8,%9, 0, 128, 1, 1 +%endm + +%macro memcpy_avx_128_1_ret 9 + __memcpy_int %1,%2,%3,%4,%5,%6,%7,%8,%9, 1, 128, 1, 1 +%endm + + +%macro memcpy_avx_16 5 + __memcpy_int %1,%2,%3,%4,%5,,,,, 0, 16, 0, 1 +%endm + +%macro memcpy_avx_16_1 5 + __memcpy_int %1,%2,%3,%4,%5,,,,, 1, 16, 0, 1 +%endm + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +%macro memcpy_avx2_64 7 + __memcpy_int %1,%2,%3,%4,%5,%6,%7,--,--, 0, 64, 0, 2 +%endm + +%macro memcpy_avx2_64_1 7 + __memcpy_int %1,%2,%3,%4,%5,%6,%7,--,--, 1, 64, 0, 2 +%endm + +%macro memcpy_avx2_128 9 + __memcpy_int %1,%2,%3,%4,%5,%6,%7, %8, %9, 0, 128, 0, 2 +%endm + +%macro memcpy_avx2_128_1 9 + __memcpy_int %1,%2,%3,%4,%5,%6,%7, %8, %9, 1, 128, 0, 2 +%endm + +%macro memcpy_avx2_64_ret 7 + __memcpy_int %1,%2,%3,%4,%5,%6,%7,--,--, 0, 64, 1, 2 +%endm + +%macro memcpy_avx2_64_1_ret 7 + __memcpy_int %1,%2,%3,%4,%5,%6,%7,--,--, 1, 64, 1, 2 +%endm + +%macro memcpy_avx2_128_ret 9 + __memcpy_int %1,%2,%3,%4,%5,%6,%7,%8,%9, 0, 128, 1, 2 +%endm + +%macro memcpy_avx2_128_1_ret 9 + __memcpy_int %1,%2,%3,%4,%5,%6,%7,%8,%9, 1, 128, 1, 2 +%endm + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + + +%macro __memcpy_int 13 +%define %%DST %1 ; register: pointer to dst (not modified) +%define %%SRC %2 ; register: pointer to src (not modified) +%define %%SIZE %3 ; register: length in bytes (not modified) +%define %%TMP0 %4 ; 64-bit temp GPR (clobbered) +%define %%TMP1 %5 ; 64-bit temp GPR (clobbered) +%define %%XTMP0 %6 ; temp XMM (clobbered) +%define %%XTMP1 %7 ; temp XMM (clobbered) +%define %%XTMP2 %8 ; temp XMM (clobbered) +%define %%XTMP3 %9 ; temp XMM (clobbered) +%define %%NOT0 %10 ; if not 0, then assume size cannot be zero +%define %%MAXSIZE %11 ; 128, 64, etc +%define %%USERET %12 ; if not 0, use "ret" at end +%define %%USEAVX %13 ; 0 = SSE, 1 = AVX1, 2 = AVX2 + +%if (%%USERET != 0) + %define %%DONE ret +%else + %define %%DONE jmp %%end +%endif + +%if (%%USEAVX != 0) + %define %%MOVDQU vmovdqu +%else + %define %%MOVDQU movdqu +%endif + +%if (%%MAXSIZE >= 128) + test %%SIZE, 64 + jz %%lt64 + %if (%%USEAVX >= 2) + %%MOVDQU %%XTMP0, [%%SRC + 0*32] + %%MOVDQU %%XTMP1, [%%SRC + 1*32] + %%MOVDQU %%XTMP2, [%%SRC + %%SIZE - 2*32] + %%MOVDQU %%XTMP3, [%%SRC + %%SIZE - 1*32] + + %%MOVDQU [%%DST + 0*32], %%XTMP0 + %%MOVDQU [%%DST + 1*32], %%XTMP1 + %%MOVDQU [%%DST + %%SIZE - 2*32], %%XTMP2 + %%MOVDQU [%%DST + %%SIZE - 1*32], %%XTMP3 + %else + %%MOVDQU %%XTMP0, [%%SRC + 0*16] + %%MOVDQU %%XTMP1, [%%SRC + 1*16] + %%MOVDQU %%XTMP2, [%%SRC + 2*16] + %%MOVDQU %%XTMP3, [%%SRC + 3*16] + %%MOVDQU [%%DST + 0*16], %%XTMP0 + %%MOVDQU [%%DST + 1*16], %%XTMP1 + %%MOVDQU [%%DST + 2*16], %%XTMP2 + %%MOVDQU [%%DST + 3*16], %%XTMP3 + + %%MOVDQU %%XTMP0, [%%SRC + %%SIZE - 4*16] + %%MOVDQU %%XTMP1, [%%SRC + %%SIZE - 3*16] + %%MOVDQU %%XTMP2, [%%SRC + %%SIZE - 2*16] + %%MOVDQU %%XTMP3, [%%SRC + %%SIZE - 1*16] + %%MOVDQU [%%DST + %%SIZE - 4*16], %%XTMP0 + %%MOVDQU [%%DST + %%SIZE - 3*16], %%XTMP1 + %%MOVDQU [%%DST + %%SIZE - 2*16], %%XTMP2 + %%MOVDQU [%%DST + %%SIZE - 1*16], %%XTMP3 + %endif + %%DONE +%endif + +%if (%%MAXSIZE >= 64) +%%lt64: + test %%SIZE, 32 + jz %%lt32 + %if (%%USEAVX >= 2) + %%MOVDQU %%XTMP0, [%%SRC + 0*32] + %%MOVDQU %%XTMP1, [%%SRC + %%SIZE - 1*32] + %%MOVDQU [%%DST + 0*32], %%XTMP0 + %%MOVDQU [%%DST + %%SIZE - 1*32], %%XTMP1 + %else + %%MOVDQU %%XTMP0, [%%SRC + 0*16] + %%MOVDQU %%XTMP1, [%%SRC + 1*16] + %%MOVDQU %%XTMP2, [%%SRC + %%SIZE - 2*16] + %%MOVDQU %%XTMP3, [%%SRC + %%SIZE - 1*16] + %%MOVDQU [%%DST + 0*16], %%XTMP0 + %%MOVDQU [%%DST + 1*16], %%XTMP1 + %%MOVDQU [%%DST + %%SIZE - 2*16], %%XTMP2 + %%MOVDQU [%%DST + %%SIZE - 1*16], %%XTMP3 + %endif + %%DONE +%endif + +%if (%%MAXSIZE >= 32) +%%lt32: + test %%SIZE, 16 + jz %%lt16 + %if (%%USEAVX >= 2) + %%MOVDQU XWORD(%%XTMP0), [%%SRC + 0*16] + %%MOVDQU XWORD(%%XTMP1), [%%SRC + %%SIZE - 1*16] + %%MOVDQU [%%DST + 0*16], XWORD(%%XTMP0) + %%MOVDQU [%%DST + %%SIZE - 1*16], XWORD(%%XTMP1) + %else + %%MOVDQU %%XTMP0, [%%SRC + 0*16] + %%MOVDQU %%XTMP1, [%%SRC + %%SIZE - 1*16] + %%MOVDQU [%%DST + 0*16], %%XTMP0 + %%MOVDQU [%%DST + %%SIZE - 1*16], %%XTMP1 + %endif + %%DONE +%endif + +%if (%%MAXSIZE >= 16) +%%lt16: + test %%SIZE, 8 + jz %%lt8 + mov %%TMP0, [%%SRC] + mov %%TMP1, [%%SRC + %%SIZE - 8] + mov [%%DST], %%TMP0 + mov [%%DST + %%SIZE - 8], %%TMP1 + %%DONE +%endif + +%if (%%MAXSIZE >= 8) +%%lt8: + test %%SIZE, 4 + jz %%lt4 + mov DWORD(%%TMP0), [%%SRC] + mov DWORD(%%TMP1), [%%SRC + %%SIZE - 4] + mov [%%DST], DWORD(%%TMP0) + mov [%%DST + %%SIZE - 4], DWORD(%%TMP1) + %%DONE +%endif + +%if (%%MAXSIZE >= 4) +%%lt4: + test %%SIZE, 2 + jz %%lt2 + movzx DWORD(%%TMP0), word [%%SRC] + movzx DWORD(%%TMP1), byte [%%SRC + %%SIZE - 1] + mov [%%DST], WORD(%%TMP0) + mov [%%DST + %%SIZE - 1], BYTE(%%TMP1) + %%DONE +%endif + +%%lt2: +%if (%%NOT0 == 0) + test %%SIZE, 1 + jz %%end +%endif + movzx DWORD(%%TMP0), byte [%%SRC] + mov [%%DST], BYTE(%%TMP0) +%%end: +%if (%%USERET != 0) + ret +%endif +%endm + +%endif ; ifndef __MEMCPY_ASM__ diff --git a/src/spdk/intel-ipsec-mb/include/os.asm b/src/spdk/intel-ipsec-mb/include/os.asm new file mode 100644 index 00000000..f54043ed --- /dev/null +++ b/src/spdk/intel-ipsec-mb/include/os.asm @@ -0,0 +1,58 @@ +;; +;; Copyright (c) 2017-2018, Intel Corporation +;; +;; Redistribution and use in source and binary forms, with or without +;; modification, are permitted provided that the following conditions are met: +;; +;; * Redistributions of source code must retain the above copyright notice, +;; this list of conditions and the following disclaimer. +;; * Redistributions in binary form must reproduce the above copyright +;; notice, this list of conditions and the following disclaimer in the +;; documentation and/or other materials provided with the distribution. +;; * Neither the name of Intel Corporation nor the names of its contributors +;; may be used to endorse or promote products derived from this software +;; without specific prior written permission. +;; +;; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +;; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +;; IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +;; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE +;; FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +;; DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +;; SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +;; CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +;; OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +;; +%ifndef OS_ASM_FILE +%define OS_ASM_FILE + +%ifndef WIN_ABI +%ifidn __OUTPUT_FORMAT__, win64 +%define WIN_ABI +%endif +%endif + +%ifndef LINUX +%ifidn __OUTPUT_FORMAT__, elf64 +%define LINUX +%endif +%endif + +%ifdef LINUX +;;; macro to declare global symbols +;;; - name : symbol name +;;; - type : funtion or data +;;; - scope : internal, private, default +%define MKGLOBAL(name,type,scope) global name %+ : %+ type scope +%endif ; LINUX + +%ifdef WIN_ABI +;;; macro to declare global symbols +;;; - name : symbol name +;;; - type : funtion or data +;;; - scope : internal, private, default (ignored in win64 coff format) +%define MKGLOBAL(name,type,scope) global name +%endif ; WIN_ABI + +%endif ; OS_ASM_FILE diff --git a/src/spdk/intel-ipsec-mb/include/reg_sizes.asm b/src/spdk/intel-ipsec-mb/include/reg_sizes.asm new file mode 100644 index 00000000..138d3995 --- /dev/null +++ b/src/spdk/intel-ipsec-mb/include/reg_sizes.asm @@ -0,0 +1,115 @@ +;; +;; Copyright (c) 2012-2018, Intel Corporation +;; +;; Redistribution and use in source and binary forms, with or without +;; modification, are permitted provided that the following conditions are met: +;; +;; * Redistributions of source code must retain the above copyright notice, +;; this list of conditions and the following disclaimer. +;; * Redistributions in binary form must reproduce the above copyright +;; notice, this list of conditions and the following disclaimer in the +;; documentation and/or other materials provided with the distribution. +;; * Neither the name of Intel Corporation nor the names of its contributors +;; may be used to endorse or promote products derived from this software +;; without specific prior written permission. +;; +;; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +;; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +;; IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +;; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE +;; FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +;; DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +;; SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +;; CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +;; OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +;; + +; define d and w variants for registers + +%define raxd eax +%define raxw ax +%define raxb al + +%define rbxd ebx +%define rbxw bx +%define rbxb bl + +%define rcxd ecx +%define rcxw cx +%define rcxb cl + +%define rdxd edx +%define rdxw dx +%define rdxb dl + +%define rsid esi +%define rsiw si +%define rsib sil + +%define rdid edi +%define rdiw di +%define rdib dil + +%define rbpd ebp +%define rbpw bp +%define rbpb bpl + +%define zmm0x xmm0 +%define zmm1x xmm1 +%define zmm2x xmm2 +%define zmm3x xmm3 +%define zmm4x xmm4 +%define zmm5x xmm5 +%define zmm6x xmm6 +%define zmm7x xmm7 +%define zmm8x xmm8 +%define zmm9x xmm9 +%define zmm10x xmm10 +%define zmm11x xmm11 +%define zmm12x xmm12 +%define zmm13x xmm13 +%define zmm14x xmm14 +%define zmm15x xmm15 + +%define ymm0x xmm0 +%define ymm1x xmm1 +%define ymm2x xmm2 +%define ymm3x xmm3 +%define ymm4x xmm4 +%define ymm5x xmm5 +%define ymm6x xmm6 +%define ymm7x xmm7 +%define ymm8x xmm8 +%define ymm9x xmm9 +%define ymm10x xmm10 +%define ymm11x xmm11 +%define ymm12x xmm12 +%define ymm13x xmm13 +%define ymm14x xmm14 +%define ymm15x xmm15 + +%define zmm0y ymm0 +%define zmm1y ymm1 +%define zmm2y ymm2 +%define zmm3y ymm3 +%define zmm4y ymm4 +%define zmm5y ymm5 +%define zmm6y ymm6 +%define zmm7y ymm7 +%define zmm8y ymm8 +%define zmm9y ymm9 +%define zmm10y ymm10 +%define zmm11y ymm11 +%define zmm12y ymm12 +%define zmm13y ymm13 +%define zmm14y ymm14 +%define zmm15y ymm15 + + +%define DWORD(reg) reg %+ d +%define WORD(reg) reg %+ w +%define BYTE(reg) reg %+ b + +%define XWORD(reg) reg %+ x +%define YWORD(reg) reg %+ y diff --git a/src/spdk/intel-ipsec-mb/include/save_xmms.asm b/src/spdk/intel-ipsec-mb/include/save_xmms.asm new file mode 100644 index 00000000..85b7799d --- /dev/null +++ b/src/spdk/intel-ipsec-mb/include/save_xmms.asm @@ -0,0 +1,102 @@ +;; +;; Copyright (c) 2012-2018, Intel Corporation +;; +;; Redistribution and use in source and binary forms, with or without +;; modification, are permitted provided that the following conditions are met: +;; +;; * Redistributions of source code must retain the above copyright notice, +;; this list of conditions and the following disclaimer. +;; * Redistributions in binary form must reproduce the above copyright +;; notice, this list of conditions and the following disclaimer in the +;; documentation and/or other materials provided with the distribution. +;; * Neither the name of Intel Corporation nor the names of its contributors +;; may be used to endorse or promote products derived from this software +;; without specific prior written permission. +;; +;; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +;; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +;; IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +;; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE +;; FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +;; DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +;; SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +;; CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +;; OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +;; + +%include "os.asm" + +%ifdef LINUX +%define ARG1 rdi +%else +%define ARG1 rcx +%endif + +section .text +; void save_xmms(UINT128 array[10]) +MKGLOBAL(save_xmms,function,internal) +save_xmms: + movdqa [ARG1 + 0*16], xmm6 + movdqa [ARG1 + 1*16], xmm7 + movdqa [ARG1 + 2*16], xmm8 + movdqa [ARG1 + 3*16], xmm9 + movdqa [ARG1 + 4*16], xmm10 + movdqa [ARG1 + 5*16], xmm11 + movdqa [ARG1 + 6*16], xmm12 + movdqa [ARG1 + 7*16], xmm13 + movdqa [ARG1 + 8*16], xmm14 + movdqa [ARG1 + 9*16], xmm15 + ret + + +; void restore_xmms(UINT128 array[10]) +MKGLOBAL(restore_xmms,function,internal) +restore_xmms: + movdqa xmm6, [ARG1 + 0*16] + movdqa xmm7, [ARG1 + 1*16] + movdqa xmm8, [ARG1 + 2*16] + movdqa xmm9, [ARG1 + 3*16] + movdqa xmm10, [ARG1 + 4*16] + movdqa xmm11, [ARG1 + 5*16] + movdqa xmm12, [ARG1 + 6*16] + movdqa xmm13, [ARG1 + 7*16] + movdqa xmm14, [ARG1 + 8*16] + movdqa xmm15, [ARG1 + 9*16] + ret + + + ; void save_xmms_avx(UINT128 array[10]) +MKGLOBAL(save_xmms_avx,function,internal) +save_xmms_avx: + vmovdqa [ARG1 + 0*16], xmm6 + vmovdqa [ARG1 + 1*16], xmm7 + vmovdqa [ARG1 + 2*16], xmm8 + vmovdqa [ARG1 + 3*16], xmm9 + vmovdqa [ARG1 + 4*16], xmm10 + vmovdqa [ARG1 + 5*16], xmm11 + vmovdqa [ARG1 + 6*16], xmm12 + vmovdqa [ARG1 + 7*16], xmm13 + vmovdqa [ARG1 + 8*16], xmm14 + vmovdqa [ARG1 + 9*16], xmm15 + ret + + +; void restore_xmms_avx(UINT128 array[10]) +MKGLOBAL(restore_xmms_avx,function,internal) +restore_xmms_avx: + vmovdqa xmm6, [ARG1 + 0*16] + vmovdqa xmm7, [ARG1 + 1*16] + vmovdqa xmm8, [ARG1 + 2*16] + vmovdqa xmm9, [ARG1 + 3*16] + vmovdqa xmm10, [ARG1 + 4*16] + vmovdqa xmm11, [ARG1 + 5*16] + vmovdqa xmm12, [ARG1 + 6*16] + vmovdqa xmm13, [ARG1 + 7*16] + vmovdqa xmm14, [ARG1 + 8*16] + vmovdqa xmm15, [ARG1 + 9*16] + ret + +%ifdef LINUX +section .note.GNU-stack noalloc noexec nowrite progbits +%endif diff --git a/src/spdk/intel-ipsec-mb/include/save_xmms.h b/src/spdk/intel-ipsec-mb/include/save_xmms.h new file mode 100644 index 00000000..213a84b0 --- /dev/null +++ b/src/spdk/intel-ipsec-mb/include/save_xmms.h @@ -0,0 +1,37 @@ +/******************************************************************************* + Copyright (c) 2012-2018, Intel Corporation + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of Intel Corporation nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE + FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*******************************************************************************/ + +#ifndef SAVE_XMMS_H +#define SAVE_XMMS_H + +void save_xmms(uint128_t array[10]); +void restore_xmms(uint128_t array[10]); + +void save_xmms_avx(uint128_t array[10]); +void restore_xmms_avx(uint128_t array[10]); + +#endif /* SAVE_XMMS_H */ -- cgit v1.2.3