summaryrefslogtreecommitdiffstats
path: root/src/spdk/intel-ipsec-mb/include
diff options
context:
space:
mode:
Diffstat (limited to '')
-rw-r--r--src/spdk/intel-ipsec-mb/include/datastruct.asm235
-rw-r--r--src/spdk/intel-ipsec-mb/include/dbgprint.asm409
-rw-r--r--src/spdk/intel-ipsec-mb/include/des_utils.h134
-rw-r--r--src/spdk/intel-ipsec-mb/include/memcpy.asm344
-rw-r--r--src/spdk/intel-ipsec-mb/include/os.asm58
-rw-r--r--src/spdk/intel-ipsec-mb/include/reg_sizes.asm115
-rw-r--r--src/spdk/intel-ipsec-mb/include/save_xmms.asm102
-rw-r--r--src/spdk/intel-ipsec-mb/include/save_xmms.h37
8 files changed, 1434 insertions, 0 deletions
diff --git a/src/spdk/intel-ipsec-mb/include/datastruct.asm b/src/spdk/intel-ipsec-mb/include/datastruct.asm
new file mode 100644
index 00000000..0ab1113a
--- /dev/null
+++ b/src/spdk/intel-ipsec-mb/include/datastruct.asm
@@ -0,0 +1,235 @@
+;;
+;; Copyright (c) 2012-2018, Intel Corporation
+;;
+;; Redistribution and use in source and binary forms, with or without
+;; modification, are permitted provided that the following conditions are met:
+;;
+;; * Redistributions of source code must retain the above copyright notice,
+;; this list of conditions and the following disclaimer.
+;; * Redistributions in binary form must reproduce the above copyright
+;; notice, this list of conditions and the following disclaimer in the
+;; documentation and/or other materials provided with the distribution.
+;; * Neither the name of Intel Corporation nor the names of its contributors
+;; may be used to endorse or promote products derived from this software
+;; without specific prior written permission.
+;;
+;; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+;; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+;; IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+;; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
+;; FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+;; DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+;; SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+;; CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+;; OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+;;
+
+; Macros for defining data structures
+
+; Usage example
+
+;START_FIELDS ; JOB_AES
+;;; name size align
+;FIELD _plaintext, 8, 8 ; pointer to plaintext
+;FIELD _ciphertext, 8, 8 ; pointer to ciphertext
+;FIELD _IV, 16, 8 ; IV
+;FIELD _keys, 8, 8 ; pointer to keys
+;FIELD _len, 4, 4 ; length in bytes
+;FIELD _status, 4, 4 ; status enumeration
+;FIELD _user_data, 8, 8 ; pointer to user data
+;UNION _union, size1, align1, \
+; size2, align2, \
+; size3, align3, \
+; ...
+;END_FIELDS
+;%assign _JOB_AES_size _FIELD_OFFSET
+;%assign _JOB_AES_align _STRUCT_ALIGN
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+; Alternate "struc-like" syntax:
+; STRUCT job_aes2
+; RES_Q .plaintext, 1
+; RES_Q .ciphertext, 1
+; RES_DQ .IV, 1
+; RES_B .nested, _JOB_AES_SIZE, _JOB_AES_ALIGN
+; RES_U .union, size1, align1, \
+; size2, align2, \
+; ...
+; ENDSTRUCT
+; ; Following only needed if nesting
+; %assign job_aes2_size _FIELD_OFFSET
+; %assign job_aes2_align _STRUCT_ALIGN
+;
+; RES_* macros take a name, a count and an optional alignment.
+; The count in in terms of the base size of the macro, and the
+; default alignment is the base size.
+; The macros are:
+; Macro Base size
+; RES_B 1
+; RES_W 2
+; RES_D 4
+; RES_Q 8
+; RES_DQ 16
+; RES_Y 32
+; RES_Z 64
+;
+; RES_U defines a union. It's arguments are a name and two or more
+; pairs of "size, alignment"
+;
+; The two assigns are only needed if this structure is being nested
+; within another. Even if the assigns are not done, one can still use
+; STRUCT_NAME_size as the size of the structure.
+;
+; Note that for nesting, you still need to assign to STRUCT_NAME_size.
+;
+; The differences between this and using "struc" directly are that each
+; type is implicitly aligned to its natural length (although this can be
+; over-ridden with an explicit third parameter), and that the structure
+; is padded at the end to its overall alignment.
+;
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+%ifndef _DATASTRUCT_ASM_
+%define _DATASTRUCT_ASM_
+
+;; START_FIELDS
+%macro START_FIELDS 0
+%assign _FIELD_OFFSET 0
+%assign _STRUCT_ALIGN 0
+%endm
+
+;; FIELD name size align
+%macro FIELD 3
+%define %%name %1
+%define %%size %2
+%define %%align %3
+
+%assign _FIELD_OFFSET (_FIELD_OFFSET + (%%align) - 1) & (~ ((%%align)-1))
+%%name equ _FIELD_OFFSET
+%assign _FIELD_OFFSET _FIELD_OFFSET + (%%size)
+%if (%%align > _STRUCT_ALIGN)
+%assign _STRUCT_ALIGN %%align
+%endif
+%endm
+
+;; END_FIELDS
+%macro END_FIELDS 0
+%assign _FIELD_OFFSET (_FIELD_OFFSET + _STRUCT_ALIGN-1) & (~ (_STRUCT_ALIGN-1))
+%endm
+
+%macro UNION 5-*
+%if (0 == (%0 & 1))
+ %error EVEN number of parameters to UNION Macro
+ %err
+%endif
+%rotate 1
+ %assign _UNION_SIZE %1
+ %assign _UNION_ALIGN %2
+%rep (%0 - 3)/2
+ %rotate 2
+ %if (%1 > _UNION_SIZE)
+ %assign _UNION_SIZE %1
+ %endif
+ %if (%2 > _UNION_ALIGN)
+ %assign _UNION_ALIGN %2
+ %endif
+%endrep
+%rotate 2
+FIELD %1, _UNION_SIZE, _UNION_ALIGN
+%endm
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+%macro STRUCT 1
+START_FIELDS
+struc %1
+%endm
+
+%macro ENDSTRUCT 0
+%assign %%tmp _FIELD_OFFSET
+END_FIELDS
+%assign %%tmp (_FIELD_OFFSET - %%tmp)
+%if (%%tmp > 0)
+ resb %%tmp
+%endif
+endstruc
+%endm
+
+;; RES_int name size align
+%macro RES_int 3
+%define %%name %1
+%define %%size %2
+%define %%align %3
+
+%assign _FIELD_OFFSET (_FIELD_OFFSET + (%%align) - 1) & (~ ((%%align)-1))
+align %%align
+%%name resb %%size
+%assign _FIELD_OFFSET _FIELD_OFFSET + (%%size)
+%if (%%align > _STRUCT_ALIGN)
+%assign _STRUCT_ALIGN %%align
+%endif
+%endm
+
+
+
+; macro RES_B name, size [, align]
+%macro RES_B 2-3 1
+RES_int %1, %2, %3
+%endm
+
+; macro RES_W name, size [, align]
+%macro RES_W 2-3 2
+RES_int %1, 2*(%2), %3
+%endm
+
+; macro RES_D name, size [, align]
+%macro RES_D 2-3 4
+RES_int %1, 4*(%2), %3
+%endm
+
+; macro RES_Q name, size [, align]
+%macro RES_Q 2-3 8
+RES_int %1, 8*(%2), %3
+%endm
+
+; macro RES_DQ name, size [, align]
+%macro RES_DQ 2-3 16
+RES_int %1, 16*(%2), %3
+%endm
+
+; macro RES_Y name, size [, align]
+%macro RES_Y 2-3 32
+RES_int %1, 32*(%2), %3
+%endm
+
+; macro RES_Z name, size [, align]
+%macro RES_Z 2-3 64
+RES_int %1, 64*(%2), %3
+%endm
+
+
+%macro RES_U 5-*
+%if (0 == (%0 & 1))
+ %error EVEN number of parameters to RES_U Macro
+ %err
+%endif
+%rotate 1
+ %assign _UNION_SIZE %1
+ %assign _UNION_ALIGN %2
+%rep (%0 - 3)/2
+ %rotate 2
+ %if (%1 > _UNION_SIZE)
+ %assign _UNION_SIZE %1
+ %endif
+ %if (%2 > _UNION_ALIGN)
+ %assign _UNION_ALIGN %2
+ %endif
+%endrep
+%rotate 2
+RES_int %1, _UNION_SIZE, _UNION_ALIGN
+%endm
+
+%endif ; end ifdef _DATASTRUCT_ASM_
diff --git a/src/spdk/intel-ipsec-mb/include/dbgprint.asm b/src/spdk/intel-ipsec-mb/include/dbgprint.asm
new file mode 100644
index 00000000..8da9c7b2
--- /dev/null
+++ b/src/spdk/intel-ipsec-mb/include/dbgprint.asm
@@ -0,0 +1,409 @@
+;;
+;; Copyright (c) 2012-2018, Intel Corporation
+;;
+;; Redistribution and use in source and binary forms, with or without
+;; modification, are permitted provided that the following conditions are met:
+;;
+;; * Redistributions of source code must retain the above copyright notice,
+;; this list of conditions and the following disclaimer.
+;; * Redistributions in binary form must reproduce the above copyright
+;; notice, this list of conditions and the following disclaimer in the
+;; documentation and/or other materials provided with the distribution.
+;; * Neither the name of Intel Corporation nor the names of its contributors
+;; may be used to endorse or promote products derived from this software
+;; without specific prior written permission.
+;;
+;; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+;; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+;; IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+;; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
+;; FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+;; DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+;; SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+;; CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+;; OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+;;
+
+; Macros for "printing" for debug purposes from within asm code
+;
+; The basic macros are:
+; DBGPRINT16, DBGPRINT32, DBGPRINT64, DBGPRINT_XMM, DBGPRINT_YMM, DBGPRINT_ZMM
+; These are called with 1 or more arguments, all of which are of the
+; size/type as specified in the name. E.g.
+; DBGPRINT64 reg1, reg2, reg3, ...
+;
+; There is also a macro DEBUGPRINTL that takes one argument, a string. E.g.
+; DBGPRINTL "hit this point in the code"
+;
+; There are also variations on these with the "DBGPRINT" suffixed with "L", e.g.
+; DBGPRINTL64. These take two or more arguments, where the first is a string,
+; and the rest are of the specified type, e.g.
+; DBGPRINTL64 "Rindex", Rindex
+; Essentially, this is the same as a DBGPRINTL followed by DBGPRINT64.
+;
+; If DO_DBGPRINT is defined, then the macros write the debug information into
+; a buffer. If DO_DBGPRINT is *not* defined, then the macros expand to nothing.
+;
+; CAVEAT: The macros need a GPR. Currently, it uses R15. If the first register
+; argument is R15, then it will use R14. This means that if you try
+; DBGPRINTL64 "text", rax, r15
+; you will not get the proper value of r15.
+; One way to avoid this issue is to not use multiple registers on the same line
+; if the register types are GPR (i.e. this is not an issue for printing XMM
+; registers). E.g the above could be done with:
+; DBGPRINTL64 "test", rax
+; DBGPRINT64 r15
+;
+; Note also that the macros only check for r15. Thus is you tried something
+; like (after token expansion):
+; DBGPRINT32 r15d
+; you won't get the right results. If you want to display r15d, you should
+; print it as the 64-bit r15.
+;
+; To actually print the data, from your C code include the file
+; "dbgprint.h". The default buffer size is 16kB. If you want to change
+; that, #define DBG_BUFFER_SIZE before including "dbgprint.h".
+;
+; Then, (after your asm routine(s) have returned, call
+; print_debug() or print_debug(file pointer)
+; If you do not specify a file pointer, it defaults to stdout.
+;
+; Printing the debug data also resets the write pointer to the beginning,
+; effectively "deleting" the previous messages.
+;
+
+;%define DO_DBGPRINT
+%ifdef DO_DBGPRINT
+extern pDebugBuffer
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+; DBGPRINT_INT size, param, ...
+%macro DBGPRINT_INT 2-*
+%ifidni %2,r15
+%xdefine %%reg r14
+%else
+%xdefine %%reg r15
+%endif
+%xdefine %%size %1
+%rotate 1
+ push %%reg
+ mov %%reg, [pDebugBuffer]
+%rep %0 - 1
+ mov byte [%%reg], %%size
+ %if (%%size == 2)
+ mov word [%%reg+1], %1
+ %elif (%%size == 4)
+ mov dword [%%reg+1], %1
+ %elif (%%size == 8)
+ mov qword [%%reg+1], %1
+ %elif (%%size == 16)
+ movdqu oword [%%reg+1], %1
+ %elif (%%size == 32)
+ vmovdqu [%%reg+1], %1
+ %elif (%%size == 64)
+ vmovdqu32 [%%reg+1], %1
+ %else
+ %error invalid size %%size
+ %endif
+ add %%reg, %%size+1
+%rotate 1
+%endrep
+ mov [pDebugBuffer], %%reg
+ pop %%reg
+%endmacro
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+; DBGPRINTL_INT size, label, param, ...
+%macro DBGPRINTL_INT 3-*
+%ifidni %3,r15
+%xdefine %%reg r14
+%else
+%xdefine %%reg r15
+%endif
+%xdefine %%size %1
+%rotate 1
+ push %%reg
+ mov %%reg, [pDebugBuffer]
+
+ mov byte [%%reg], 0x57
+section .data
+%%lab: db %1, 0
+section .text
+ mov qword [%%reg+1], %%lab
+ add %%reg, 8+1
+%rotate 1
+
+%rep %0 - 2
+ mov byte [%%reg], %%size
+%if (%%size == 2)
+ mov word [%%reg+1], %1
+%elif (%%size == 4)
+ mov dword [%%reg+1], %1
+%elif (%%size == 8)
+ mov qword [%%reg+1], %1
+%elif (%%size == 16)
+ movdqu oword [%%reg+1], %1
+%elif (%%size == 32)
+ vmovdqu [%%reg+1], %1
+%elif (%%size == 64)
+ vmovdqu32 [%%reg+1], %1
+%else
+%error invalid size %%size
+%endif
+ add %%reg, %%size+1
+%rotate 1
+%endrep
+ mov [pDebugBuffer], %%reg
+ pop %%reg
+%endmacro
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+; DBGPRINTL* data, ...
+%macro DBGPRINT16 1+
+ DBGPRINT_INT 2, %1
+%endmacro
+%macro DBGPRINT32 1+
+ DBGPRINT_INT 4, %1
+%endmacro
+%macro DBGPRINT64 1+
+ DBGPRINT_INT 8, %1
+%endmacro
+%macro DBGPRINT_XMM 1+
+ DBGPRINT_INT 16, %1
+%endmacro
+%macro DBGPRINT_YMM 1+
+ DBGPRINT_INT 32, %1
+%endmacro
+%macro DBGPRINT_ZMM 1+
+ DBGPRINT_INT 64, %1
+%endmacro
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+; DBGPRINTL* label, data, ...
+%macro DBGPRINTL16 2+
+ DBGPRINTL_INT 2, %1, %2
+%endmacro
+%macro DBGPRINTL32 2+
+ DBGPRINTL_INT 4, %1, %2
+%endmacro
+%macro DBGPRINTL64 2+
+ DBGPRINTL_INT 8, %1, %2
+%endmacro
+%macro DBGPRINTL_XMM 2+
+ DBGPRINTL_INT 16, %1, %2
+%endmacro
+%macro DBGPRINTL_YMM 2+
+ DBGPRINTL_INT 32, %1, %2
+%endmacro
+%macro DBGPRINTL_ZMM 2+
+ DBGPRINTL_INT 64, %1, %2
+%endmacro
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+%macro DBGPRINTL 1
+ push r15
+ mov r15, [pDebugBuffer]
+
+ mov byte [r15], 0x57
+section .data
+%%lab: db %1, 0
+section .text
+ mov qword [r15+1], %%lab
+ add r15, 8+1
+
+ mov [pDebugBuffer], r15
+ pop r15
+%endmacro
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+%else
+%macro DBGPRINT16 1+
+%endmacro
+%macro DBGPRINT32 1+
+%endmacro
+%macro DBGPRINT64 1+
+%endmacro
+%macro DBGPRINT_XMM 1+
+%endmacro
+%macro DBGPRINT_YMM 1+
+%endmacro
+%macro DBGPRINT_ZMM 1+
+%endmacro
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+%macro DBGPRINTL16 2+
+%endmacro
+%macro DBGPRINTL32 2+
+%endmacro
+%macro DBGPRINTL64 2+
+%endmacro
+%macro DBGPRINTL_XMM 2+
+%endmacro
+%macro DBGPRINTL_YMM 2+
+%endmacro
+%macro DBGPRINTL_ZMM 2+
+%endmacro
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+%macro DBGPRINTL 1
+%endmacro
+%endif
+
+
+
+%if 0 ; OLD
+%macro DBGPRINTL_ZMM 2-*
+ push rax
+ mov rax, [pDebugBuffer]
+
+ mov byte [rax], 0x57
+section .data
+%%lab: db %1, 0
+section .text
+ mov qword [rax+1], %%lab
+ add rax, 8+1
+%rotate 1
+
+%rep %0 - 1
+ mov byte [rax], 64
+ vmovdqu32 [rax+1], %1
+%rotate 1
+ add rax, 64+1
+%endrep
+ mov [pDebugBuffer], rax
+ pop rax
+%endmacro
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+%macro DBGPRINT_ZMM 1-*
+ push rax
+ mov rax, [pDebugBuffer]
+%rep %0
+ mov byte [rax], 64
+ vmovdqu32 [rax+1], %1
+%rotate 1
+ add rax, 64+1
+%endrep
+ mov [pDebugBuffer], rax
+ pop rax
+%endmacro
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+%macro DBGPRINT_YMM 1-*
+ push rax
+ mov rax, [pDebugBuffer]
+%rep %0
+ mov byte [rax], 32
+ vmovdqu [rax+1], %1
+%rotate 1
+ add rax, 32+1
+%endrep
+ mov [pDebugBuffer], rax
+ pop rax
+%endmacro
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+%macro DBGPRINT_XMM 1-*
+ push rax
+ mov rax, [pDebugBuffer]
+%rep %0
+ mov byte [rax], 16
+ vmovdqu oword [rax+1], %1
+%rotate 1
+ add rax, 16+1
+%endrep
+ mov [pDebugBuffer], rax
+ pop rax
+%endmacro
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+%macro DBGPRINTL64 2-*
+ push rax
+ mov rax, [pDebugBuffer]
+
+ mov byte [rax], 0x57
+section .data
+%%lab: db %1, 0
+section .text
+ mov qword [rax+1], %%lab
+ add rax, 8+1
+%rotate 1
+
+%rep %0 - 1
+ mov byte [rax], 8
+ mov qword [rax+1], %1
+%rotate 1
+ add rax, 8+1
+%endrep
+ mov [pDebugBuffer], rax
+ pop rax
+%endmacro
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+%macro DBGPRINT64 1-*
+ push rax
+ mov rax, [pDebugBuffer]
+%rep %0
+ mov byte [rax], 8
+ mov qword [rax+1], %1
+%rotate 1
+ add rax, 8+1
+%endrep
+ mov [pDebugBuffer], rax
+ pop rax
+%endmacro
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+%macro DBGPRINT32 1-*
+ push rax
+ mov rax, [pDebugBuffer]
+%rep %0
+ mov byte [rax], 4
+ mov dword [rax+1], %1
+%rotate 1
+ add rax, 4+1
+%endrep
+ mov [pDebugBuffer], rax
+ pop rax
+%endmacro
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+%macro DBGPRINT16 1-*
+ push rax
+ mov rax, [pDebugBuffer]
+%rep %0
+ mov byte [rax], 2
+ mov word [rax+1], %1
+%rotate 1
+ add rax, 2+1
+%endrep
+ mov [pDebugBuffer], rax
+ pop rax
+%endmacro
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+%macro DBGPRINT_LAB 1
+ push rax
+ mov rax, [pDebugBuffer]
+
+ mov byte [rax], 0x57
+section .data
+%%lab: db %1, 0
+section .text
+ mov qword [rax+1], %%lab
+ add rax, 8+1
+
+ mov [pDebugBuffer], rax
+ pop rax
+%endmacro
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+%macro DBGHIST 2
+ inc dword [%1 + 4 * %2]
+%endmacro
+%macro DBGPRINT_ZMM 1-*
+%endmacro
+%macro DBGPRINT_YMM 1-*
+%endmacro
+%macro DBGPRINT_XMM 1-*
+%endmacro
+%macro DBGPRINT64 1-*
+%endmacro
+%macro DBGPRINT32 1-*
+%endmacro
+%macro DBGPRINT16 1-*
+%endmacro
+%macro DBGHIST 2
+%endmacro
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+%endif ; ifdef 0 ; OLD
diff --git a/src/spdk/intel-ipsec-mb/include/des_utils.h b/src/spdk/intel-ipsec-mb/include/des_utils.h
new file mode 100644
index 00000000..4358132d
--- /dev/null
+++ b/src/spdk/intel-ipsec-mb/include/des_utils.h
@@ -0,0 +1,134 @@
+/*******************************************************************************
+ Copyright (c) 2017-2018, Intel Corporation
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions are met:
+
+ * Redistributions of source code must retain the above copyright notice,
+ this list of conditions and the following disclaimer.
+ * Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+ * Neither the name of Intel Corporation nor the names of its contributors
+ may be used to endorse or promote products derived from this software
+ without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
+ FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*******************************************************************************/
+
+/* DES utility functions and macros */
+
+#ifndef DES_UTILS_H
+#define DES_UTILS_H
+
+#include <stdint.h>
+#include "intel-ipsec-mb.h"
+
+/**
+ * @brief Gets selected bit value out of a 64-bit word
+ *
+ * @param val 64-bit word
+ * @param n bit number (0 to 63) to get value of
+ *
+ * @return n-th bit value (0 or 1 value only)
+ */
+__forceinline
+uint64_t bit_get64b(const uint64_t val, const unsigned n)
+{
+ IMB_ASSERT(n < 64);
+ return (val >> n) & UINT64_C(1);
+}
+
+/**
+ * @brief Sets selected bit in a 64-bit word
+ *
+ * @param val 64-bit word
+ * @param n bit number (0 to 63) to get value of
+ * @param b bit value (0 or 1)
+ *
+ * @return val with n-th bit set to value b
+ */
+__forceinline
+uint64_t bit_set64b(const uint64_t val, const unsigned n, const uint64_t b)
+{
+ const uint64_t m = UINT64_C(1) << n;
+
+ IMB_ASSERT(n < 64);
+ return (val & (~m)) | (b << n);
+}
+
+/**
+ * @brief Permutes bits in a 64-bit word as described by pattern
+ *
+ * The function goes through pattern array from index 0 to 'size' (max 63).
+ * It sets output bit number 'index' to value of
+ * bit number 'pattern[index] - 1' from 'in'.
+ *
+ * @param in 64-bit word to be permuted
+ * @param pattern pointer to array defining the permutation
+ * @param size is size of the permutation pattern
+ *
+ * @return permuted in word as described by the pattern
+ */
+__forceinline
+uint64_t permute_64b(const uint64_t in, const uint8_t *pattern, const int size)
+{
+ uint64_t out = 0;
+ int n = 0;
+
+ IMB_ASSERT(size <= 64);
+
+ for (n = 0; n < size; n++) {
+ /* '-1' is required as bit numbers in FIPS start with 1 not 0 */
+ const int m = ((int) pattern[n]) - 1;
+ const uint64_t bit_val = bit_get64b(in, m);
+
+ out = bit_set64b(out, n, bit_val);
+ }
+
+ return out;
+}
+
+static const uint8_t reflect_tab[16] = {
+ /* [ 0] 0000 => 0000 */ 0, /* [ 1] 0001 => 1000 */ 8,
+ /* [ 2] 0010 => 0100 */ 4, /* [ 3] 0011 => 1100 */ 12,
+ /* [ 4] 0100 => 0010 */ 2, /* [ 5] 0101 => 1010 */ 10,
+ /* [ 6] 0110 => 0110 */ 6, /* [ 7] 0111 => 1110 */ 14,
+ /* [ 8] 1000 => 0001 */ 1, /* [ 9] 1001 => 1001 */ 9,
+ /* [10] 1010 => 0101 */ 5, /* [11] 1011 => 1101 */ 13,
+ /* [12] 1100 => 0011 */ 3, /* [13] 1101 => 1011 */ 11,
+ /* [14] 1110 => 0111 */ 7, /* [15] 1111 => 1111 */ 15
+};
+
+__forceinline
+uint8_t reflect_8b(const uint8_t pb)
+{
+ return reflect_tab[pb >> 4] | (reflect_tab[pb & 15] << 4);
+}
+
+__forceinline
+uint64_t load64_reflect(const void *key)
+{
+ const uint8_t *kb = (const uint8_t *) key;
+
+ return ((uint64_t) reflect_8b(kb[0])) |
+ ((uint64_t) reflect_8b(kb[1])) << 8 |
+ ((uint64_t) reflect_8b(kb[2])) << 16 |
+ ((uint64_t) reflect_8b(kb[3])) << 24 |
+ ((uint64_t) reflect_8b(kb[4])) << 32 |
+ ((uint64_t) reflect_8b(kb[5])) << 40 |
+ ((uint64_t) reflect_8b(kb[6])) << 48 |
+ ((uint64_t) reflect_8b(kb[7])) << 56;
+}
+
+
+#endif /* DES_UTILS_H */
diff --git a/src/spdk/intel-ipsec-mb/include/memcpy.asm b/src/spdk/intel-ipsec-mb/include/memcpy.asm
new file mode 100644
index 00000000..0844ea8a
--- /dev/null
+++ b/src/spdk/intel-ipsec-mb/include/memcpy.asm
@@ -0,0 +1,344 @@
+;;
+;; Copyright (c) 2012-2018, Intel Corporation
+;;
+;; Redistribution and use in source and binary forms, with or without
+;; modification, are permitted provided that the following conditions are met:
+;;
+;; * Redistributions of source code must retain the above copyright notice,
+;; this list of conditions and the following disclaimer.
+;; * Redistributions in binary form must reproduce the above copyright
+;; notice, this list of conditions and the following disclaimer in the
+;; documentation and/or other materials provided with the distribution.
+;; * Neither the name of Intel Corporation nor the names of its contributors
+;; may be used to endorse or promote products derived from this software
+;; without specific prior written permission.
+;;
+;; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+;; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+;; IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+;; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
+;; FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+;; DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+;; SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+;; CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+;; OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+;;
+
+%ifndef __MEMCPY_ASM__
+%define __MEMCPY_ASM__
+
+%include "reg_sizes.asm"
+
+
+; This file defines a series of macros to copy small to medium amounts
+; of data from memory to memory, where the size is variable but limited.
+;
+; The macros are all called as:
+; memcpy DST, SRC, SIZE, TMP0, TMP1, XTMP0, XTMP1, XTMP2, XTMP3
+; with the parameters defined as:
+; DST : register: pointer to dst (not modified)
+; SRC : register: pointer to src (not modified)
+; SIZE : register: length in bytes (not modified)
+; TMP0 : 64-bit temp GPR (clobbered)
+; TMP1 : 64-bit temp GPR (clobbered)
+; XTMP0 : temp XMM (clobbered)
+; XTMP1 : temp XMM (clobbered)
+; XTMP2 : temp XMM (clobbered)
+; XTMP3 : temp XMM (clobbered)
+;
+; The name indicates the options. The name is of the form:
+; memcpy_<VEC>_<SZ><ZERO><RET>
+; where:
+; <VEC> is either "sse" or "avx" or "avx2"
+; <SZ> is either "64" or "128" and defines largest value of SIZE
+; <ZERO> is blank or "_1". If "_1" then the min SIZE is 1 (otherwise 0)
+; <RET> is blank or "_ret". If blank, the code falls through. If "ret"
+; it does a "ret" at the end
+;
+; For the avx2 versions, the temp XMM registers need to be YMM registers
+; If the SZ is 64, then only two YMM temps are needed, i.e. it is called as:
+; memcpy_avx2_64 DST, SRC, SIZE, TMP0, TMP1, YTMP0, YTMP1
+; memcpy_avx2_128 DST, SRC, SIZE, TMP0, TMP1, YTMP0, YTMP1, YTMP2, YTMP3
+;
+; For example:
+; memcpy_sse_64 : SSE, 0 <= size < 64, falls through
+; memcpy_avx_64_1 : AVX1, 1 <= size < 64, falls through
+; memcpy_sse_128_ret : SSE, 0 <= size < 128, ends with ret
+; mempcy_avx_128_1_ret : AVX1, 1 <= size < 128, ends with ret
+;
+
+%macro memcpy_sse_64 9
+ __memcpy_int %1,%2,%3,%4,%5,%6,%7,%8,%9, 0, 64, 0, 0
+%endm
+
+%macro memcpy_sse_64_1 9
+ __memcpy_int %1,%2,%3,%4,%5,%6,%7,%8,%9, 1, 64, 0, 0
+%endm
+
+%macro memcpy_sse_128 9
+ __memcpy_int %1,%2,%3,%4,%5,%6,%7,%8,%9, 0, 128, 0, 0
+%endm
+
+%macro memcpy_sse_128_1 9
+ __memcpy_int %1,%2,%3,%4,%5,%6,%7,%8,%9, 1, 128, 0, 0
+%endm
+
+%macro memcpy_sse_64_ret 9
+ __memcpy_int %1,%2,%3,%4,%5,%6,%7,%8,%9, 0, 64, 1, 0
+%endm
+
+%macro memcpy_sse_64_1_ret 9
+ __memcpy_int %1,%2,%3,%4,%5,%6,%7,%8,%9, 1, 64, 1, 0
+%endm
+
+%macro memcpy_sse_128_ret 9
+ __memcpy_int %1,%2,%3,%4,%5,%6,%7,%8,%9, 0, 128, 1, 0
+%endm
+
+%macro memcpy_sse_128_1_ret 9
+ __memcpy_int %1,%2,%3,%4,%5,%6,%7,%8,%9, 1, 128, 1, 0
+%endm
+
+
+%macro memcpy_sse_16 5
+ __memcpy_int %1,%2,%3,%4,%5,,,,, 0, 16, 0, 0
+%endm
+
+%macro memcpy_sse_16_1 5
+ __memcpy_int %1,%2,%3,%4,%5,,,,, 1, 16, 0, 0
+%endm
+
+ ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+%macro memcpy_avx_64 9
+ __memcpy_int %1,%2,%3,%4,%5,%6,%7,%8,%9, 0, 64, 0, 1
+%endm
+
+%macro memcpy_avx_64_1 9
+ __memcpy_int %1,%2,%3,%4,%5,%6,%7,%8,%9, 1, 64, 0, 1
+%endm
+
+%macro memcpy_avx_128 9
+ __memcpy_int %1,%2,%3,%4,%5,%6,%7,%8,%9, 0, 128, 0, 1
+%endm
+
+%macro memcpy_avx_128_1 9
+ __memcpy_int %1,%2,%3,%4,%5,%6,%7,%8,%9, 1, 128, 0, 1
+%endm
+
+%macro memcpy_avx_64_ret 9
+ __memcpy_int %1,%2,%3,%4,%5,%6,%7,%8,%9, 0, 64, 1, 1
+%endm
+
+%macro memcpy_avx_64_1_ret 9
+ __memcpy_int %1,%2,%3,%4,%5,%6,%7,%8,%9, 1, 64, 1, 1
+%endm
+
+%macro memcpy_avx_128_ret 9
+ __memcpy_int %1,%2,%3,%4,%5,%6,%7,%8,%9, 0, 128, 1, 1
+%endm
+
+%macro memcpy_avx_128_1_ret 9
+ __memcpy_int %1,%2,%3,%4,%5,%6,%7,%8,%9, 1, 128, 1, 1
+%endm
+
+
+%macro memcpy_avx_16 5
+ __memcpy_int %1,%2,%3,%4,%5,,,,, 0, 16, 0, 1
+%endm
+
+%macro memcpy_avx_16_1 5
+ __memcpy_int %1,%2,%3,%4,%5,,,,, 1, 16, 0, 1
+%endm
+
+ ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+%macro memcpy_avx2_64 7
+ __memcpy_int %1,%2,%3,%4,%5,%6,%7,--,--, 0, 64, 0, 2
+%endm
+
+%macro memcpy_avx2_64_1 7
+ __memcpy_int %1,%2,%3,%4,%5,%6,%7,--,--, 1, 64, 0, 2
+%endm
+
+%macro memcpy_avx2_128 9
+ __memcpy_int %1,%2,%3,%4,%5,%6,%7, %8, %9, 0, 128, 0, 2
+%endm
+
+%macro memcpy_avx2_128_1 9
+ __memcpy_int %1,%2,%3,%4,%5,%6,%7, %8, %9, 1, 128, 0, 2
+%endm
+
+%macro memcpy_avx2_64_ret 7
+ __memcpy_int %1,%2,%3,%4,%5,%6,%7,--,--, 0, 64, 1, 2
+%endm
+
+%macro memcpy_avx2_64_1_ret 7
+ __memcpy_int %1,%2,%3,%4,%5,%6,%7,--,--, 1, 64, 1, 2
+%endm
+
+%macro memcpy_avx2_128_ret 9
+ __memcpy_int %1,%2,%3,%4,%5,%6,%7,%8,%9, 0, 128, 1, 2
+%endm
+
+%macro memcpy_avx2_128_1_ret 9
+ __memcpy_int %1,%2,%3,%4,%5,%6,%7,%8,%9, 1, 128, 1, 2
+%endm
+
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+
+%macro __memcpy_int 13
+%define %%DST %1 ; register: pointer to dst (not modified)
+%define %%SRC %2 ; register: pointer to src (not modified)
+%define %%SIZE %3 ; register: length in bytes (not modified)
+%define %%TMP0 %4 ; 64-bit temp GPR (clobbered)
+%define %%TMP1 %5 ; 64-bit temp GPR (clobbered)
+%define %%XTMP0 %6 ; temp XMM (clobbered)
+%define %%XTMP1 %7 ; temp XMM (clobbered)
+%define %%XTMP2 %8 ; temp XMM (clobbered)
+%define %%XTMP3 %9 ; temp XMM (clobbered)
+%define %%NOT0 %10 ; if not 0, then assume size cannot be zero
+%define %%MAXSIZE %11 ; 128, 64, etc
+%define %%USERET %12 ; if not 0, use "ret" at end
+%define %%USEAVX %13 ; 0 = SSE, 1 = AVX1, 2 = AVX2
+
+%if (%%USERET != 0)
+ %define %%DONE ret
+%else
+ %define %%DONE jmp %%end
+%endif
+
+%if (%%USEAVX != 0)
+ %define %%MOVDQU vmovdqu
+%else
+ %define %%MOVDQU movdqu
+%endif
+
+%if (%%MAXSIZE >= 128)
+ test %%SIZE, 64
+ jz %%lt64
+ %if (%%USEAVX >= 2)
+ %%MOVDQU %%XTMP0, [%%SRC + 0*32]
+ %%MOVDQU %%XTMP1, [%%SRC + 1*32]
+ %%MOVDQU %%XTMP2, [%%SRC + %%SIZE - 2*32]
+ %%MOVDQU %%XTMP3, [%%SRC + %%SIZE - 1*32]
+
+ %%MOVDQU [%%DST + 0*32], %%XTMP0
+ %%MOVDQU [%%DST + 1*32], %%XTMP1
+ %%MOVDQU [%%DST + %%SIZE - 2*32], %%XTMP2
+ %%MOVDQU [%%DST + %%SIZE - 1*32], %%XTMP3
+ %else
+ %%MOVDQU %%XTMP0, [%%SRC + 0*16]
+ %%MOVDQU %%XTMP1, [%%SRC + 1*16]
+ %%MOVDQU %%XTMP2, [%%SRC + 2*16]
+ %%MOVDQU %%XTMP3, [%%SRC + 3*16]
+ %%MOVDQU [%%DST + 0*16], %%XTMP0
+ %%MOVDQU [%%DST + 1*16], %%XTMP1
+ %%MOVDQU [%%DST + 2*16], %%XTMP2
+ %%MOVDQU [%%DST + 3*16], %%XTMP3
+
+ %%MOVDQU %%XTMP0, [%%SRC + %%SIZE - 4*16]
+ %%MOVDQU %%XTMP1, [%%SRC + %%SIZE - 3*16]
+ %%MOVDQU %%XTMP2, [%%SRC + %%SIZE - 2*16]
+ %%MOVDQU %%XTMP3, [%%SRC + %%SIZE - 1*16]
+ %%MOVDQU [%%DST + %%SIZE - 4*16], %%XTMP0
+ %%MOVDQU [%%DST + %%SIZE - 3*16], %%XTMP1
+ %%MOVDQU [%%DST + %%SIZE - 2*16], %%XTMP2
+ %%MOVDQU [%%DST + %%SIZE - 1*16], %%XTMP3
+ %endif
+ %%DONE
+%endif
+
+%if (%%MAXSIZE >= 64)
+%%lt64:
+ test %%SIZE, 32
+ jz %%lt32
+ %if (%%USEAVX >= 2)
+ %%MOVDQU %%XTMP0, [%%SRC + 0*32]
+ %%MOVDQU %%XTMP1, [%%SRC + %%SIZE - 1*32]
+ %%MOVDQU [%%DST + 0*32], %%XTMP0
+ %%MOVDQU [%%DST + %%SIZE - 1*32], %%XTMP1
+ %else
+ %%MOVDQU %%XTMP0, [%%SRC + 0*16]
+ %%MOVDQU %%XTMP1, [%%SRC + 1*16]
+ %%MOVDQU %%XTMP2, [%%SRC + %%SIZE - 2*16]
+ %%MOVDQU %%XTMP3, [%%SRC + %%SIZE - 1*16]
+ %%MOVDQU [%%DST + 0*16], %%XTMP0
+ %%MOVDQU [%%DST + 1*16], %%XTMP1
+ %%MOVDQU [%%DST + %%SIZE - 2*16], %%XTMP2
+ %%MOVDQU [%%DST + %%SIZE - 1*16], %%XTMP3
+ %endif
+ %%DONE
+%endif
+
+%if (%%MAXSIZE >= 32)
+%%lt32:
+ test %%SIZE, 16
+ jz %%lt16
+ %if (%%USEAVX >= 2)
+ %%MOVDQU XWORD(%%XTMP0), [%%SRC + 0*16]
+ %%MOVDQU XWORD(%%XTMP1), [%%SRC + %%SIZE - 1*16]
+ %%MOVDQU [%%DST + 0*16], XWORD(%%XTMP0)
+ %%MOVDQU [%%DST + %%SIZE - 1*16], XWORD(%%XTMP1)
+ %else
+ %%MOVDQU %%XTMP0, [%%SRC + 0*16]
+ %%MOVDQU %%XTMP1, [%%SRC + %%SIZE - 1*16]
+ %%MOVDQU [%%DST + 0*16], %%XTMP0
+ %%MOVDQU [%%DST + %%SIZE - 1*16], %%XTMP1
+ %endif
+ %%DONE
+%endif
+
+%if (%%MAXSIZE >= 16)
+%%lt16:
+ test %%SIZE, 8
+ jz %%lt8
+ mov %%TMP0, [%%SRC]
+ mov %%TMP1, [%%SRC + %%SIZE - 8]
+ mov [%%DST], %%TMP0
+ mov [%%DST + %%SIZE - 8], %%TMP1
+ %%DONE
+%endif
+
+%if (%%MAXSIZE >= 8)
+%%lt8:
+ test %%SIZE, 4
+ jz %%lt4
+ mov DWORD(%%TMP0), [%%SRC]
+ mov DWORD(%%TMP1), [%%SRC + %%SIZE - 4]
+ mov [%%DST], DWORD(%%TMP0)
+ mov [%%DST + %%SIZE - 4], DWORD(%%TMP1)
+ %%DONE
+%endif
+
+%if (%%MAXSIZE >= 4)
+%%lt4:
+ test %%SIZE, 2
+ jz %%lt2
+ movzx DWORD(%%TMP0), word [%%SRC]
+ movzx DWORD(%%TMP1), byte [%%SRC + %%SIZE - 1]
+ mov [%%DST], WORD(%%TMP0)
+ mov [%%DST + %%SIZE - 1], BYTE(%%TMP1)
+ %%DONE
+%endif
+
+%%lt2:
+%if (%%NOT0 == 0)
+ test %%SIZE, 1
+ jz %%end
+%endif
+ movzx DWORD(%%TMP0), byte [%%SRC]
+ mov [%%DST], BYTE(%%TMP0)
+%%end:
+%if (%%USERET != 0)
+ ret
+%endif
+%endm
+
+%endif ; ifndef __MEMCPY_ASM__
diff --git a/src/spdk/intel-ipsec-mb/include/os.asm b/src/spdk/intel-ipsec-mb/include/os.asm
new file mode 100644
index 00000000..f54043ed
--- /dev/null
+++ b/src/spdk/intel-ipsec-mb/include/os.asm
@@ -0,0 +1,58 @@
+;;
+;; Copyright (c) 2017-2018, Intel Corporation
+;;
+;; Redistribution and use in source and binary forms, with or without
+;; modification, are permitted provided that the following conditions are met:
+;;
+;; * Redistributions of source code must retain the above copyright notice,
+;; this list of conditions and the following disclaimer.
+;; * Redistributions in binary form must reproduce the above copyright
+;; notice, this list of conditions and the following disclaimer in the
+;; documentation and/or other materials provided with the distribution.
+;; * Neither the name of Intel Corporation nor the names of its contributors
+;; may be used to endorse or promote products derived from this software
+;; without specific prior written permission.
+;;
+;; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+;; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+;; IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+;; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
+;; FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+;; DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+;; SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+;; CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+;; OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+;;
+%ifndef OS_ASM_FILE
+%define OS_ASM_FILE
+
+%ifndef WIN_ABI
+%ifidn __OUTPUT_FORMAT__, win64
+%define WIN_ABI
+%endif
+%endif
+
+%ifndef LINUX
+%ifidn __OUTPUT_FORMAT__, elf64
+%define LINUX
+%endif
+%endif
+
+%ifdef LINUX
+;;; macro to declare global symbols
+;;; - name : symbol name
+;;; - type : funtion or data
+;;; - scope : internal, private, default
+%define MKGLOBAL(name,type,scope) global name %+ : %+ type scope
+%endif ; LINUX
+
+%ifdef WIN_ABI
+;;; macro to declare global symbols
+;;; - name : symbol name
+;;; - type : funtion or data
+;;; - scope : internal, private, default (ignored in win64 coff format)
+%define MKGLOBAL(name,type,scope) global name
+%endif ; WIN_ABI
+
+%endif ; OS_ASM_FILE
diff --git a/src/spdk/intel-ipsec-mb/include/reg_sizes.asm b/src/spdk/intel-ipsec-mb/include/reg_sizes.asm
new file mode 100644
index 00000000..138d3995
--- /dev/null
+++ b/src/spdk/intel-ipsec-mb/include/reg_sizes.asm
@@ -0,0 +1,115 @@
+;;
+;; Copyright (c) 2012-2018, Intel Corporation
+;;
+;; Redistribution and use in source and binary forms, with or without
+;; modification, are permitted provided that the following conditions are met:
+;;
+;; * Redistributions of source code must retain the above copyright notice,
+;; this list of conditions and the following disclaimer.
+;; * Redistributions in binary form must reproduce the above copyright
+;; notice, this list of conditions and the following disclaimer in the
+;; documentation and/or other materials provided with the distribution.
+;; * Neither the name of Intel Corporation nor the names of its contributors
+;; may be used to endorse or promote products derived from this software
+;; without specific prior written permission.
+;;
+;; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+;; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+;; IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+;; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
+;; FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+;; DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+;; SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+;; CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+;; OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+;;
+
+; define d and w variants for registers
+
+%define raxd eax
+%define raxw ax
+%define raxb al
+
+%define rbxd ebx
+%define rbxw bx
+%define rbxb bl
+
+%define rcxd ecx
+%define rcxw cx
+%define rcxb cl
+
+%define rdxd edx
+%define rdxw dx
+%define rdxb dl
+
+%define rsid esi
+%define rsiw si
+%define rsib sil
+
+%define rdid edi
+%define rdiw di
+%define rdib dil
+
+%define rbpd ebp
+%define rbpw bp
+%define rbpb bpl
+
+%define zmm0x xmm0
+%define zmm1x xmm1
+%define zmm2x xmm2
+%define zmm3x xmm3
+%define zmm4x xmm4
+%define zmm5x xmm5
+%define zmm6x xmm6
+%define zmm7x xmm7
+%define zmm8x xmm8
+%define zmm9x xmm9
+%define zmm10x xmm10
+%define zmm11x xmm11
+%define zmm12x xmm12
+%define zmm13x xmm13
+%define zmm14x xmm14
+%define zmm15x xmm15
+
+%define ymm0x xmm0
+%define ymm1x xmm1
+%define ymm2x xmm2
+%define ymm3x xmm3
+%define ymm4x xmm4
+%define ymm5x xmm5
+%define ymm6x xmm6
+%define ymm7x xmm7
+%define ymm8x xmm8
+%define ymm9x xmm9
+%define ymm10x xmm10
+%define ymm11x xmm11
+%define ymm12x xmm12
+%define ymm13x xmm13
+%define ymm14x xmm14
+%define ymm15x xmm15
+
+%define zmm0y ymm0
+%define zmm1y ymm1
+%define zmm2y ymm2
+%define zmm3y ymm3
+%define zmm4y ymm4
+%define zmm5y ymm5
+%define zmm6y ymm6
+%define zmm7y ymm7
+%define zmm8y ymm8
+%define zmm9y ymm9
+%define zmm10y ymm10
+%define zmm11y ymm11
+%define zmm12y ymm12
+%define zmm13y ymm13
+%define zmm14y ymm14
+%define zmm15y ymm15
+
+
+%define DWORD(reg) reg %+ d
+%define WORD(reg) reg %+ w
+%define BYTE(reg) reg %+ b
+
+%define XWORD(reg) reg %+ x
+%define YWORD(reg) reg %+ y
diff --git a/src/spdk/intel-ipsec-mb/include/save_xmms.asm b/src/spdk/intel-ipsec-mb/include/save_xmms.asm
new file mode 100644
index 00000000..85b7799d
--- /dev/null
+++ b/src/spdk/intel-ipsec-mb/include/save_xmms.asm
@@ -0,0 +1,102 @@
+;;
+;; Copyright (c) 2012-2018, Intel Corporation
+;;
+;; Redistribution and use in source and binary forms, with or without
+;; modification, are permitted provided that the following conditions are met:
+;;
+;; * Redistributions of source code must retain the above copyright notice,
+;; this list of conditions and the following disclaimer.
+;; * Redistributions in binary form must reproduce the above copyright
+;; notice, this list of conditions and the following disclaimer in the
+;; documentation and/or other materials provided with the distribution.
+;; * Neither the name of Intel Corporation nor the names of its contributors
+;; may be used to endorse or promote products derived from this software
+;; without specific prior written permission.
+;;
+;; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+;; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+;; IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+;; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
+;; FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+;; DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+;; SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+;; CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+;; OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+;;
+
+%include "os.asm"
+
+%ifdef LINUX
+%define ARG1 rdi
+%else
+%define ARG1 rcx
+%endif
+
+section .text
+; void save_xmms(UINT128 array[10])
+MKGLOBAL(save_xmms,function,internal)
+save_xmms:
+ movdqa [ARG1 + 0*16], xmm6
+ movdqa [ARG1 + 1*16], xmm7
+ movdqa [ARG1 + 2*16], xmm8
+ movdqa [ARG1 + 3*16], xmm9
+ movdqa [ARG1 + 4*16], xmm10
+ movdqa [ARG1 + 5*16], xmm11
+ movdqa [ARG1 + 6*16], xmm12
+ movdqa [ARG1 + 7*16], xmm13
+ movdqa [ARG1 + 8*16], xmm14
+ movdqa [ARG1 + 9*16], xmm15
+ ret
+
+
+; void restore_xmms(UINT128 array[10])
+MKGLOBAL(restore_xmms,function,internal)
+restore_xmms:
+ movdqa xmm6, [ARG1 + 0*16]
+ movdqa xmm7, [ARG1 + 1*16]
+ movdqa xmm8, [ARG1 + 2*16]
+ movdqa xmm9, [ARG1 + 3*16]
+ movdqa xmm10, [ARG1 + 4*16]
+ movdqa xmm11, [ARG1 + 5*16]
+ movdqa xmm12, [ARG1 + 6*16]
+ movdqa xmm13, [ARG1 + 7*16]
+ movdqa xmm14, [ARG1 + 8*16]
+ movdqa xmm15, [ARG1 + 9*16]
+ ret
+
+
+ ; void save_xmms_avx(UINT128 array[10])
+MKGLOBAL(save_xmms_avx,function,internal)
+save_xmms_avx:
+ vmovdqa [ARG1 + 0*16], xmm6
+ vmovdqa [ARG1 + 1*16], xmm7
+ vmovdqa [ARG1 + 2*16], xmm8
+ vmovdqa [ARG1 + 3*16], xmm9
+ vmovdqa [ARG1 + 4*16], xmm10
+ vmovdqa [ARG1 + 5*16], xmm11
+ vmovdqa [ARG1 + 6*16], xmm12
+ vmovdqa [ARG1 + 7*16], xmm13
+ vmovdqa [ARG1 + 8*16], xmm14
+ vmovdqa [ARG1 + 9*16], xmm15
+ ret
+
+
+; void restore_xmms_avx(UINT128 array[10])
+MKGLOBAL(restore_xmms_avx,function,internal)
+restore_xmms_avx:
+ vmovdqa xmm6, [ARG1 + 0*16]
+ vmovdqa xmm7, [ARG1 + 1*16]
+ vmovdqa xmm8, [ARG1 + 2*16]
+ vmovdqa xmm9, [ARG1 + 3*16]
+ vmovdqa xmm10, [ARG1 + 4*16]
+ vmovdqa xmm11, [ARG1 + 5*16]
+ vmovdqa xmm12, [ARG1 + 6*16]
+ vmovdqa xmm13, [ARG1 + 7*16]
+ vmovdqa xmm14, [ARG1 + 8*16]
+ vmovdqa xmm15, [ARG1 + 9*16]
+ ret
+
+%ifdef LINUX
+section .note.GNU-stack noalloc noexec nowrite progbits
+%endif
diff --git a/src/spdk/intel-ipsec-mb/include/save_xmms.h b/src/spdk/intel-ipsec-mb/include/save_xmms.h
new file mode 100644
index 00000000..213a84b0
--- /dev/null
+++ b/src/spdk/intel-ipsec-mb/include/save_xmms.h
@@ -0,0 +1,37 @@
+/*******************************************************************************
+ Copyright (c) 2012-2018, Intel Corporation
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions are met:
+
+ * Redistributions of source code must retain the above copyright notice,
+ this list of conditions and the following disclaimer.
+ * Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+ * Neither the name of Intel Corporation nor the names of its contributors
+ may be used to endorse or promote products derived from this software
+ without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
+ FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*******************************************************************************/
+
+#ifndef SAVE_XMMS_H
+#define SAVE_XMMS_H
+
+void save_xmms(uint128_t array[10]);
+void restore_xmms(uint128_t array[10]);
+
+void save_xmms_avx(uint128_t array[10]);
+void restore_xmms_avx(uint128_t array[10]);
+
+#endif /* SAVE_XMMS_H */
generated by cgit v1.2.3 (git 2.39.1) at 2024-12-24 15:44:46 +0000