diff options
Diffstat (limited to 'src/spdk/intel-ipsec-mb/avx512')
33 files changed, 19771 insertions, 0 deletions
diff --git a/src/spdk/intel-ipsec-mb/avx512/aes_cbc_dec_vaes_avx512.asm b/src/spdk/intel-ipsec-mb/avx512/aes_cbc_dec_vaes_avx512.asm new file mode 100644 index 000000000..ce33caa92 --- /dev/null +++ b/src/spdk/intel-ipsec-mb/avx512/aes_cbc_dec_vaes_avx512.asm @@ -0,0 +1,477 @@ +;; +;; Copyright (c) 2019, Intel Corporation +;; +;; Redistribution and use in source and binary forms, with or without +;; modification, are permitted provided that the following conditions are met: +;; +;; * Redistributions of source code must retain the above copyright notice, +;; this list of conditions and the following disclaimer. +;; * Redistributions in binary form must reproduce the above copyright +;; notice, this list of conditions and the following disclaimer in the +;; documentation and/or other materials provided with the distribution. +;; * Neither the name of Intel Corporation nor the names of its contributors +;; may be used to endorse or promote products derived from this software +;; without specific prior written permission. +;; +;; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +;; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +;; IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +;; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE +;; FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +;; DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +;; SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +;; CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +;; OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +;; + +%include "include/os.asm" +%include "include/reg_sizes.asm" +%include "include/aes_common.asm" + +%define zIV zmm0 +%define zBLK_0_3 zmm1 +%define zBLK_4_7 zmm2 +%define zBLK_8_11 zmm3 +%define zBLK_12_15 zmm4 +%define zTMP0 zmm5 +%define zTMP1 zmm6 +%define zTMP2 zmm7 +%define zTMP3 zmm8 + +%define ZKEY0 zmm17 +%define ZKEY1 zmm18 +%define ZKEY2 zmm19 +%define ZKEY3 zmm20 +%define ZKEY4 zmm21 +%define ZKEY5 zmm22 +%define ZKEY6 zmm23 +%define ZKEY7 zmm24 +%define ZKEY8 zmm25 +%define ZKEY9 zmm26 +%define ZKEY10 zmm27 +%define ZKEY11 zmm28 +%define ZKEY12 zmm29 +%define ZKEY13 zmm30 +%define ZKEY14 zmm31 + +%ifdef LINUX +%define p_in rdi +%define p_IV rsi +%define p_keys rdx +%define p_out rcx +%define num_bytes r8 +%else +%define p_in rcx +%define p_IV rdx +%define p_keys r8 +%define p_out r9 +%define num_bytes rax +%endif + +%define tmp r10 + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;; macro to preload keys +;;; - uses ZKEY[0-14] registers (ZMM) +%macro LOAD_KEYS 2 +%define %%KEYS %1 ; [in] key pointer +%define %%NROUNDS %2 ; [in] numerical value, number of AES rounds + ; excluding 1st and last rounds. + ; Example: AES-128 -> value 9 + +%assign i 0 +%rep (%%NROUNDS + 2) + vbroadcastf64x2 ZKEY %+ i, [%%KEYS + 16*i] +%assign i (i + 1) +%endrep + +%endmacro + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;; This macro is used to "cool down" pipeline after DECRYPT_16_PARALLEL macro +;;; code as the number of final blocks is variable. +;;; Processes the last %%num_final_blocks blocks (1 to 15, can't be 0) + +%macro FINAL_BLOCKS 14 +%define %%PLAIN_OUT %1 ; [in] output buffer +%define %%CIPH_IN %2 ; [in] input buffer +%define %%LAST_CIPH_BLK %3 ; [in/out] ZMM with IV/last cipher blk (in idx 3) +%define %%num_final_blocks %4 ; [in] numerical value (1 - 15) +%define %%CIPHER_PLAIN_0_3 %5 ; [out] ZMM next 0-3 cipher blocks +%define %%CIPHER_PLAIN_4_7 %6 ; [out] ZMM next 4-7 cipher blocks +%define %%CIPHER_PLAIN_8_11 %7 ; [out] ZMM next 8-11 cipher blocks +%define %%CIPHER_PLAIN_12_15 %8 ; [out] ZMM next 12-15 cipher blocks +%define %%ZT1 %9 ; [clobbered] ZMM temporary +%define %%ZT2 %10 ; [clobbered] ZMM temporary +%define %%ZT3 %11 ; [clobbered] ZMM temporary +%define %%ZT4 %12 ; [clobbered] ZMM temporary +%define %%IA0 %13 ; [clobbered] GP temporary +%define %%NROUNDS %14 ; [in] number of rounds; numerical value + + ;; load plain/cipher text + ZMM_LOAD_BLOCKS_0_16 %%num_final_blocks, %%CIPH_IN, 0, \ + %%CIPHER_PLAIN_0_3, %%CIPHER_PLAIN_4_7, \ + %%CIPHER_PLAIN_8_11, %%CIPHER_PLAIN_12_15 + + ;; Prepare final cipher text blocks to + ;; be XOR'd later after AESDEC + valignq %%ZT1, %%CIPHER_PLAIN_0_3, %%LAST_CIPH_BLK, 6 +%if %%num_final_blocks > 4 + valignq %%ZT2, %%CIPHER_PLAIN_4_7, %%CIPHER_PLAIN_0_3, 6 +%endif +%if %%num_final_blocks > 8 + valignq %%ZT3, %%CIPHER_PLAIN_8_11, %%CIPHER_PLAIN_4_7, 6 +%endif +%if %%num_final_blocks > 12 + valignq %%ZT4, %%CIPHER_PLAIN_12_15, %%CIPHER_PLAIN_8_11, 6 +%endif + + ;; Update IV with last cipher block + ;; to be used later in DECRYPT_16_PARALLEL +%if %%num_final_blocks == 1 + valignq %%LAST_CIPH_BLK, %%CIPHER_PLAIN_0_3, %%CIPHER_PLAIN_0_3, 2 +%elif %%num_final_blocks == 2 + valignq %%LAST_CIPH_BLK, %%CIPHER_PLAIN_0_3, %%CIPHER_PLAIN_0_3, 4 +%elif %%num_final_blocks == 3 + valignq %%LAST_CIPH_BLK, %%CIPHER_PLAIN_0_3, %%CIPHER_PLAIN_0_3, 6 +%elif %%num_final_blocks == 4 + vmovdqa64 %%LAST_CIPH_BLK, %%CIPHER_PLAIN_0_3 +%elif %%num_final_blocks == 5 + valignq %%LAST_CIPH_BLK, %%CIPHER_PLAIN_4_7, %%CIPHER_PLAIN_4_7, 2 +%elif %%num_final_blocks == 6 + valignq %%LAST_CIPH_BLK, %%CIPHER_PLAIN_4_7, %%CIPHER_PLAIN_4_7, 4 +%elif %%num_final_blocks == 7 + valignq %%LAST_CIPH_BLK, %%CIPHER_PLAIN_4_7, %%CIPHER_PLAIN_4_7, 6 +%elif %%num_final_blocks == 8 + vmovdqa64 %%LAST_CIPH_BLK, %%CIPHER_PLAIN_4_7 +%elif %%num_final_blocks == 9 + valignq %%LAST_CIPH_BLK, %%CIPHER_PLAIN_8_11, %%CIPHER_PLAIN_8_11, 2 +%elif %%num_final_blocks == 10 + valignq %%LAST_CIPH_BLK, %%CIPHER_PLAIN_8_11, %%CIPHER_PLAIN_8_11, 4 +%elif %%num_final_blocks == 11 + valignq %%LAST_CIPH_BLK, %%CIPHER_PLAIN_8_11, %%CIPHER_PLAIN_8_11, 6 +%elif %%num_final_blocks == 12 + vmovdqa64 %%LAST_CIPH_BLK, %%CIPHER_PLAIN_8_11 +%elif %%num_final_blocks == 13 + valignq %%LAST_CIPH_BLK, %%CIPHER_PLAIN_12_15, %%CIPHER_PLAIN_12_15, 2 +%elif %%num_final_blocks == 14 + valignq %%LAST_CIPH_BLK, %%CIPHER_PLAIN_12_15, %%CIPHER_PLAIN_12_15, 4 +%elif %%num_final_blocks == 15 + valignq %%LAST_CIPH_BLK, %%CIPHER_PLAIN_12_15, %%CIPHER_PLAIN_12_15, 6 +%endif + + ;; AES rounds +%assign j 0 +%rep (%%NROUNDS + 2) + ZMM_AESDEC_ROUND_BLOCKS_0_16 %%CIPHER_PLAIN_0_3, %%CIPHER_PLAIN_4_7, \ + %%CIPHER_PLAIN_8_11, %%CIPHER_PLAIN_12_15, \ + ZKEY %+ j, j, no_data, no_data, no_data, no_data, \ + %%num_final_blocks, %%NROUNDS +%assign j (j + 1) +%endrep + + ;; XOR with decrypted blocks to get plain text + vpxorq %%CIPHER_PLAIN_0_3, %%CIPHER_PLAIN_0_3, %%ZT1 +%if %%num_final_blocks > 4 + vpxorq %%CIPHER_PLAIN_4_7, %%CIPHER_PLAIN_4_7, %%ZT2 +%endif +%if %%num_final_blocks > 8 + vpxorq %%CIPHER_PLAIN_8_11, %%CIPHER_PLAIN_8_11, %%ZT3 +%endif +%if %%num_final_blocks > 12 + vpxorq %%CIPHER_PLAIN_12_15, %%CIPHER_PLAIN_12_15, %%ZT4 +%endif + + ;; write plain text back to output + ZMM_STORE_BLOCKS_0_16 %%num_final_blocks, %%PLAIN_OUT, 0, \ + %%CIPHER_PLAIN_0_3, %%CIPHER_PLAIN_4_7, \ + %%CIPHER_PLAIN_8_11, %%CIPHER_PLAIN_12_15 + +%endmacro ; FINAL_BLOCKS + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;; Main AES-CBC decrypt macro +;;; - operates on single stream +;;; - decrypts 16 blocks at a time +%macro DECRYPT_16_PARALLEL 14 +%define %%PLAIN_OUT %1 ; [in] output buffer +%define %%CIPH_IN %2 ; [in] input buffer +%define %%LENGTH %3 ; [in/out] number of bytes to process +%define %%LAST_CIPH_BLK %4 ; [in/out] ZMM with IV (first block) or last cipher block (idx 3) +%define %%CIPHER_PLAIN_0_3 %5 ; [out] ZMM next 0-3 cipher blocks +%define %%CIPHER_PLAIN_4_7 %6 ; [out] ZMM next 4-7 cipher blocks +%define %%CIPHER_PLAIN_8_11 %7 ; [out] ZMM next 8-11 cipher blocks +%define %%CIPHER_PLAIN_12_15 %8 ; [out] ZMM next 12-15 cipher blocks +%define %%ZT1 %9 ; [clobbered] ZMM temporary +%define %%ZT2 %10 ; [clobbered] ZMM temporary +%define %%ZT3 %11 ; [clobbered] ZMM temporary +%define %%ZT4 %12 ; [clobbered] ZMM temporary +%define %%NROUNDS %13 ; [in] number of rounds; numerical value +%define %%IA0 %14 ; [clobbered] GP temporary + + vmovdqu8 %%CIPHER_PLAIN_0_3, [%%CIPH_IN] + vmovdqu8 %%CIPHER_PLAIN_4_7, [%%CIPH_IN + 64] + vmovdqu8 %%CIPHER_PLAIN_8_11, [%%CIPH_IN + 128] + vmovdqu8 %%CIPHER_PLAIN_12_15, [%%CIPH_IN + 192] + + ;; prepare first set of cipher blocks for later XOR'ing + valignq %%ZT1, %%CIPHER_PLAIN_0_3, %%LAST_CIPH_BLK, 6 + valignq %%ZT2, %%CIPHER_PLAIN_4_7, %%CIPHER_PLAIN_0_3, 6 + valignq %%ZT3, %%CIPHER_PLAIN_8_11, %%CIPHER_PLAIN_4_7, 6 + valignq %%ZT4, %%CIPHER_PLAIN_12_15, %%CIPHER_PLAIN_8_11, 6 + + ;; store last cipher text block to be used for next 16 blocks + vmovdqa64 %%LAST_CIPH_BLK, %%CIPHER_PLAIN_12_15 + + ;; AES rounds +%assign j 0 +%rep (%%NROUNDS + 2) + ZMM_AESDEC_ROUND_BLOCKS_0_16 %%CIPHER_PLAIN_0_3, %%CIPHER_PLAIN_4_7, \ + %%CIPHER_PLAIN_8_11, %%CIPHER_PLAIN_12_15, \ + ZKEY %+ j, j, no_data, no_data, no_data, no_data, \ + 16, %%NROUNDS +%assign j (j + 1) +%endrep + + ;; XOR with decrypted blocks to get plain text + vpxorq %%CIPHER_PLAIN_0_3, %%CIPHER_PLAIN_0_3, %%ZT1 + vpxorq %%CIPHER_PLAIN_4_7, %%CIPHER_PLAIN_4_7, %%ZT2 + vpxorq %%CIPHER_PLAIN_8_11, %%CIPHER_PLAIN_8_11, %%ZT3 + vpxorq %%CIPHER_PLAIN_12_15, %%CIPHER_PLAIN_12_15, %%ZT4 + + ;; write plain text back to output + vmovdqu8 [%%PLAIN_OUT], %%CIPHER_PLAIN_0_3 + vmovdqu8 [%%PLAIN_OUT + 64], %%CIPHER_PLAIN_4_7 + vmovdqu8 [%%PLAIN_OUT + 128], %%CIPHER_PLAIN_8_11 + vmovdqu8 [%%PLAIN_OUT + 192], %%CIPHER_PLAIN_12_15 + + ;; adjust input pointer and length + sub %%LENGTH, (16 * 16) + add %%CIPH_IN, (16 * 16) + add %%PLAIN_OUT, (16 * 16) + +%endmacro ; DECRYPT_16_PARALLEL + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;; AES_CBC_DEC macro decrypts given data. +;;; Flow: +;;; - Decrypt all blocks (multiple of 16) up to final 1-15 blocks +;;; - Decrypt final blocks (1-15 blocks) +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +%macro AES_CBC_DEC 7 +%define %%CIPH_IN %1 ;; [in] pointer to input buffer +%define %%PLAIN_OUT %2 ;; [in] pointer to output buffer +%define %%KEYS %3 ;; [in] pointer to expanded keys +%define %%IV %4 ;; [in] pointer to IV +%define %%LENGTH %5 ;; [in/out] GP register with length in bytes +%define %%NROUNDS %6 ;; [in] Number of AES rounds; numerical value +%define %%TMP %7 ;; [clobbered] GP register + + cmp %%LENGTH, 0 + je %%cbc_dec_done + + vinserti64x2 zIV, zIV, [%%IV], 3 + + ;; preload keys + LOAD_KEYS %%KEYS, %%NROUNDS + +%%decrypt_16_parallel: + cmp %%LENGTH, 256 + jb %%final_blocks + + DECRYPT_16_PARALLEL %%PLAIN_OUT, %%CIPH_IN, %%LENGTH, zIV, \ + zBLK_0_3, zBLK_4_7, zBLK_8_11, zBLK_12_15, \ + zTMP0, zTMP1, zTMP2, zTMP3, %%NROUNDS, %%TMP + jmp %%decrypt_16_parallel + +%%final_blocks: + ;; get num final blocks + shr %%LENGTH, 4 + and %%LENGTH, 0xf + je %%cbc_dec_done + + cmp %%LENGTH, 8 + je %%final_num_blocks_is_8 + jl %%final_blocks_is_1_7 + + ; Final blocks 9-15 + cmp %%LENGTH, 12 + je %%final_num_blocks_is_12 + jl %%final_blocks_is_9_11 + + ; Final blocks 13-15 + cmp %%LENGTH, 15 + je %%final_num_blocks_is_15 + cmp %%LENGTH, 14 + je %%final_num_blocks_is_14 + cmp %%LENGTH, 13 + je %%final_num_blocks_is_13 + +%%final_blocks_is_9_11: + cmp %%LENGTH, 11 + je %%final_num_blocks_is_11 + cmp %%LENGTH, 10 + je %%final_num_blocks_is_10 + cmp %%LENGTH, 9 + je %%final_num_blocks_is_9 + +%%final_blocks_is_1_7: + cmp %%LENGTH, 4 + je %%final_num_blocks_is_4 + jl %%final_blocks_is_1_3 + + ; Final blocks 5-7 + cmp %%LENGTH, 7 + je %%final_num_blocks_is_7 + cmp %%LENGTH, 6 + je %%final_num_blocks_is_6 + cmp %%LENGTH, 5 + je %%final_num_blocks_is_5 + +%%final_blocks_is_1_3: + cmp %%LENGTH, 3 + je %%final_num_blocks_is_3 + cmp %%LENGTH, 2 + je %%final_num_blocks_is_2 + jmp %%final_num_blocks_is_1 + + +%%final_num_blocks_is_15: + FINAL_BLOCKS %%PLAIN_OUT, %%CIPH_IN, zIV, 15, zBLK_0_3, zBLK_4_7, \ + zBLK_8_11, zBLK_12_15, zTMP0, zTMP1, zTMP2, zTMP3, \ + %%TMP, %%NROUNDS + jmp %%cbc_dec_done + +%%final_num_blocks_is_14: + FINAL_BLOCKS %%PLAIN_OUT, %%CIPH_IN, zIV, 14, zBLK_0_3, zBLK_4_7, \ + zBLK_8_11, zBLK_12_15, zTMP0, zTMP1, zTMP2, zTMP3, \ + %%TMP, %%NROUNDS + jmp %%cbc_dec_done + +%%final_num_blocks_is_13: + FINAL_BLOCKS %%PLAIN_OUT, %%CIPH_IN, zIV, 13, zBLK_0_3, zBLK_4_7, \ + zBLK_8_11, zBLK_12_15, zTMP0, zTMP1, zTMP2, zTMP3, \ + %%TMP, %%NROUNDS + jmp %%cbc_dec_done + +%%final_num_blocks_is_12: + FINAL_BLOCKS %%PLAIN_OUT, %%CIPH_IN, zIV, 12, zBLK_0_3, zBLK_4_7, \ + zBLK_8_11, zBLK_12_15, zTMP0, zTMP1, zTMP2, zTMP3, \ + %%TMP, %%NROUNDS + jmp %%cbc_dec_done + +%%final_num_blocks_is_11: + FINAL_BLOCKS %%PLAIN_OUT, %%CIPH_IN, zIV, 11, zBLK_0_3, zBLK_4_7, \ + zBLK_8_11, zBLK_12_15, zTMP0, zTMP1, zTMP2, zTMP3, \ + %%TMP, %%NROUNDS + jmp %%cbc_dec_done + +%%final_num_blocks_is_10: + FINAL_BLOCKS %%PLAIN_OUT, %%CIPH_IN, zIV, 10, zBLK_0_3, zBLK_4_7, \ + zBLK_8_11, zBLK_12_15, zTMP0, zTMP1, zTMP2, zTMP3, \ + %%TMP, %%NROUNDS + jmp %%cbc_dec_done + +%%final_num_blocks_is_9: + FINAL_BLOCKS %%PLAIN_OUT, %%CIPH_IN, zIV, 9, zBLK_0_3, zBLK_4_7, \ + zBLK_8_11, zBLK_12_15, zTMP0, zTMP1, zTMP2, zTMP3, \ + %%TMP, %%NROUNDS + jmp %%cbc_dec_done + +%%final_num_blocks_is_8: + FINAL_BLOCKS %%PLAIN_OUT, %%CIPH_IN, zIV, 8, zBLK_0_3, zBLK_4_7, \ + zBLK_8_11, zBLK_12_15, zTMP0, zTMP1, zTMP2, zTMP3, \ + %%TMP, %%NROUNDS + jmp %%cbc_dec_done + +%%final_num_blocks_is_7: + FINAL_BLOCKS %%PLAIN_OUT, %%CIPH_IN, zIV, 7, zBLK_0_3, zBLK_4_7, \ + zBLK_8_11, zBLK_12_15, zTMP0, zTMP1, zTMP2, zTMP3, \ + %%TMP, %%NROUNDS + jmp %%cbc_dec_done + +%%final_num_blocks_is_6: + FINAL_BLOCKS %%PLAIN_OUT, %%CIPH_IN, zIV, 6, zBLK_0_3, zBLK_4_7, \ + zBLK_8_11, zBLK_12_15, zTMP0, zTMP1, zTMP2, zTMP3, \ + %%TMP, %%NROUNDS + jmp %%cbc_dec_done + +%%final_num_blocks_is_5: + FINAL_BLOCKS %%PLAIN_OUT, %%CIPH_IN, zIV, 5, zBLK_0_3, zBLK_4_7, \ + zBLK_8_11, zBLK_12_15, zTMP0, zTMP1, zTMP2, zTMP3, \ + %%TMP, %%NROUNDS + jmp %%cbc_dec_done + +%%final_num_blocks_is_4: + FINAL_BLOCKS %%PLAIN_OUT, %%CIPH_IN, zIV, 4, zBLK_0_3, zBLK_4_7, \ + zBLK_8_11, zBLK_12_15, zTMP0, zTMP1, zTMP2, zTMP3, \ + %%TMP, %%NROUNDS + jmp %%cbc_dec_done + +%%final_num_blocks_is_3: + FINAL_BLOCKS %%PLAIN_OUT, %%CIPH_IN, zIV, 3, zBLK_0_3, zBLK_4_7, \ + zBLK_8_11, zBLK_12_15, zTMP0, zTMP1, zTMP2, zTMP3, \ + %%TMP, %%NROUNDS + jmp %%cbc_dec_done + +%%final_num_blocks_is_2: + FINAL_BLOCKS %%PLAIN_OUT, %%CIPH_IN, zIV, 2, zBLK_0_3, zBLK_4_7, \ + zBLK_8_11, zBLK_12_15, zTMP0, zTMP1, zTMP2, zTMP3, \ + %%TMP, %%NROUNDS + jmp %%cbc_dec_done + +%%final_num_blocks_is_1: + FINAL_BLOCKS %%PLAIN_OUT, %%CIPH_IN, zIV, 1, zBLK_0_3, zBLK_4_7, \ + zBLK_8_11, zBLK_12_15, zTMP0, zTMP1, zTMP2, zTMP3, \ + %%TMP, %%NROUNDS + +%%cbc_dec_done: +%endmacro + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +section .text + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; aes_cbc_dec_128_vaes_avx512(void *in, void *IV, void *keys, void *out, UINT64 num_bytes) +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +MKGLOBAL(aes_cbc_dec_128_vaes_avx512,function,internal) +aes_cbc_dec_128_vaes_avx512: +%ifndef LINUX + mov num_bytes, [rsp + 8*5] +%endif + AES_CBC_DEC p_in, p_out, p_keys, p_IV, num_bytes, 9, tmp + + ret + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; aes_cbc_dec_192_vaes_avx512(void *in, void *IV, void *keys, void *out, UINT64 num_bytes) +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +MKGLOBAL(aes_cbc_dec_192_vaes_avx512,function,internal) +aes_cbc_dec_192_vaes_avx512: +%ifndef LINUX + mov num_bytes, [rsp + 8*5] +%endif + AES_CBC_DEC p_in, p_out, p_keys, p_IV, num_bytes, 11, tmp + + ret + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; aes_cbc_dec_256_vaes_avx512(void *in, void *IV, void *keys, void *out, UINT64 num_bytes) +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +MKGLOBAL(aes_cbc_dec_256_vaes_avx512,function,internal) +aes_cbc_dec_256_vaes_avx512: +%ifndef LINUX + mov num_bytes, [rsp + 8*5] +%endif + AES_CBC_DEC p_in, p_out, p_keys, p_IV, num_bytes, 13, tmp + + ret + +%ifdef LINUX +section .note.GNU-stack noalloc noexec nowrite progbits +%endif + diff --git a/src/spdk/intel-ipsec-mb/avx512/aes_cbc_enc_vaes_avx512.asm b/src/spdk/intel-ipsec-mb/avx512/aes_cbc_enc_vaes_avx512.asm new file mode 100644 index 000000000..c4b1dd561 --- /dev/null +++ b/src/spdk/intel-ipsec-mb/avx512/aes_cbc_enc_vaes_avx512.asm @@ -0,0 +1,727 @@ +;; +;; Copyright (c) 2019, Intel Corporation +;; +;; Redistribution and use in source and binary forms, with or without +;; modification, are permitted provided that the following conditions are met: +;; +;; * Redistributions of source code must retain the above copyright notice, +;; this list of conditions and the following disclaimer. +;; * Redistributions in binary form must reproduce the above copyright +;; notice, this list of conditions and the following disclaimer in the +;; documentation and/or other materials provided with the distribution. +;; * Neither the name of Intel Corporation nor the names of its contributors +;; may be used to endorse or promote products derived from this software +;; without specific prior written permission. +;; +;; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +;; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +;; IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +;; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE +;; FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +;; DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +;; SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +;; CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +;; OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +;; + +;;; routines to do 128/192/256 bit CBC AES encrypt + +%include "include/os.asm" +%include "mb_mgr_datastruct.asm" +%include "include/reg_sizes.asm" + +struc STACK +_gpr_save: resq 3 +endstruc + +%define GPR_SAVE_AREA rsp + _gpr_save + +%ifdef LINUX +%define arg1 rdi +%define arg2 rsi +%define arg3 rcx +%define arg4 rdx +%else +%define arg1 rcx +%define arg2 rdx +%define arg3 rdi +%define arg4 rsi +%endif + +%define ARG arg1 +%define LEN arg2 + +%define IA0 rax +%define IA1 rbx +%define IA2 arg3 +%define IA3 arg4 +%define IA4 rbp +%define IA5 r8 +%define IA6 r9 +%define IA7 r10 +%define IA8 r11 +%define IA9 r13 +%define IA10 r14 +%define IA11 r15 +%define IA12 r12 + +%define ZIV00_03 zmm8 +%define ZIV04_07 zmm9 +%define ZIV08_11 zmm10 +%define ZIV12_15 zmm11 + +%define ZT0 zmm16 +%define ZT1 zmm17 +%define ZT2 zmm18 +%define ZT3 zmm19 +%define ZT4 zmm20 +%define ZT5 zmm21 +%define ZT6 zmm22 +%define ZT7 zmm23 +%define ZT8 zmm24 +%define ZT9 zmm25 +%define ZT10 zmm26 +%define ZT11 zmm27 +%define ZT12 zmm28 +%define ZT13 zmm29 +%define ZT14 zmm30 +%define ZT15 zmm31 + +%define ZT16 zmm12 +%define ZT17 zmm13 +%define ZT18 zmm14 +%define ZT19 zmm15 + +%define TAB_A0B0A1B1 zmm6 +%define TAB_A2B2A3B3 zmm7 + +;; Save registers states +%macro FUNC_SAVE 0 + sub rsp, STACK_size + mov [GPR_SAVE_AREA + 8*0], rbp +%ifndef LINUX + mov [GPR_SAVE_AREA + 8*1], rsi + mov [GPR_SAVE_AREA + 8*2], rdi +%endif +%endmacro + +;; Restore register states +%macro FUNC_RESTORE 0 + ;; XMMs are saved at a higher level + mov rbp, [GPR_SAVE_AREA + 8*0] +%ifndef LINUX + mov rsi, [GPR_SAVE_AREA + 8*1] + mov rdi, [GPR_SAVE_AREA + 8*2] +%endif + add rsp, STACK_size + vzeroupper +%endmacro + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; Transpose macro - executes 4x4 transpose of 4 ZMM registers +; in: L0B0-3 out: B0L0-3 +; L1B0-3 B1L0-3 +; L2B0-3 B2L0-3 +; L3B0-3 B3L0-3 +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +%macro TRANSPOSE_4x4 8 +%define %%IN_OUT_0 %1 +%define %%IN_OUT_1 %2 +%define %%IN_OUT_2 %3 +%define %%IN_OUT_3 %4 +%define %%ZTMP_0 %5 +%define %%ZTMP_1 %6 +%define %%ZTMP_2 %7 +%define %%ZTMP_3 %8 + + vmovdqa64 %%ZTMP_0, TAB_A0B0A1B1 + vmovdqa64 %%ZTMP_1, %%ZTMP_0 + vmovdqa64 %%ZTMP_2, TAB_A2B2A3B3 + vmovdqa64 %%ZTMP_3, %%ZTMP_2 + + vpermi2q %%ZTMP_0, %%IN_OUT_0, %%IN_OUT_1 + vpermi2q %%ZTMP_1, %%IN_OUT_2, %%IN_OUT_3 + vpermi2q %%ZTMP_2, %%IN_OUT_0, %%IN_OUT_1 + vpermi2q %%ZTMP_3, %%IN_OUT_2, %%IN_OUT_3 + + vshufi64x2 %%IN_OUT_0, %%ZTMP_0, %%ZTMP_1, 0x44 + vshufi64x2 %%IN_OUT_2, %%ZTMP_2, %%ZTMP_3, 0x44 + vshufi64x2 %%IN_OUT_1, %%ZTMP_0, %%ZTMP_1, 0xee + vshufi64x2 %%IN_OUT_3, %%ZTMP_2, %%ZTMP_3, 0xee +%endmacro + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; LOAD_STORE - loads/stores 1-4 blocks (16 bytes) for 4 lanes into ZMM registers +; - Loads 4 blocks by default +; - Pass %%MASK_REG argument to load/store 1-3 blocks (optional) +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +%macro LOAD_STORE_x4 15-16 +%define %%LANE_A %1 ; [in] lane index to load/store (numerical) +%define %%LANE_B %2 ; [in] lane index to load/store (numerical) +%define %%LANE_C %3 ; [in] lane index to load/store (numerical) +%define %%LANE_D %4 ; [in] lane index to load/store (numerical) +%define %%DATA_PTR %5 ; [in] GP reg with ptr to lane input table +%define %%OFFSET %6 ; [in] GP reg input/output buffer offset +%define %%ZDATA0 %7 ; [in/out] ZMM reg to load/store data +%define %%ZDATA1 %8 ; [in/out] ZMM reg to load/store data +%define %%ZDATA2 %9 ; [in/out] ZMM reg to load/store data +%define %%ZDATA3 %10 ; [in/out] ZMM reg to load/store data +%define %%GP0 %11 ; [clobbered] tmp GP reg +%define %%GP1 %12 ; [clobbered] tmp GP reg +%define %%GP2 %13 ; [clobbered] tmp GP reg +%define %%GP3 %14 ; [clobbered] tmp GP reg +%define %%LOAD_STORE %15 ; [in] string value to select LOAD or STORE +%define %%MASK_REG %16 ; [in] mask reg used for load/store mask +%define %%NUM_ARGS %0 + + mov %%GP0, [%%DATA_PTR + 8*(%%LANE_A)] + mov %%GP1, [%%DATA_PTR + 8*(%%LANE_B)] + mov %%GP2, [%%DATA_PTR + 8*(%%LANE_C)] + mov %%GP3, [%%DATA_PTR + 8*(%%LANE_D)] + +%if %%NUM_ARGS <= 15 ;; %%MASK_REG not set, assume 4 block load/store +%ifidn %%LOAD_STORE, LOAD + vmovdqu8 %%ZDATA0, [%%GP0 + %%OFFSET] + vmovdqu8 %%ZDATA1, [%%GP1 + %%OFFSET] + vmovdqu8 %%ZDATA2, [%%GP2 + %%OFFSET] + vmovdqu8 %%ZDATA3, [%%GP3 + %%OFFSET] +%else ; STORE8 + vmovdqu8 [%%GP0 + %%OFFSET], %%ZDATA0 + vmovdqu8 [%%GP1 + %%OFFSET], %%ZDATA1 + vmovdqu8 [%%GP2 + %%OFFSET], %%ZDATA2 + vmovdqu8 [%%GP3 + %%OFFSET], %%ZDATA3 +%endif +%else ;; %%MASK_REG argument passed - 1, 2, or 3 block load/store +%ifidn %%LOAD_STORE, LOAD + vmovdqu8 %%ZDATA0{%%MASK_REG}{z}, [%%GP0 + %%OFFSET] + vmovdqu8 %%ZDATA1{%%MASK_REG}{z}, [%%GP1 + %%OFFSET] + vmovdqu8 %%ZDATA2{%%MASK_REG}{z}, [%%GP2 + %%OFFSET] + vmovdqu8 %%ZDATA3{%%MASK_REG}{z}, [%%GP3 + %%OFFSET] +%else ; STORE + vmovdqu8 [%%GP0 + %%OFFSET]{%%MASK_REG}, %%ZDATA0 + vmovdqu8 [%%GP1 + %%OFFSET]{%%MASK_REG}, %%ZDATA1 + vmovdqu8 [%%GP2 + %%OFFSET]{%%MASK_REG}, %%ZDATA2 + vmovdqu8 [%%GP3 + %%OFFSET]{%%MASK_REG}, %%ZDATA3 +%endif +%endif ;; %%NUM_ARGS +%endmacro + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; AESENC_ROUNDS_x16 macro +; - 16 lanes, 1 block per lane +; - it handles special cases: the last and zero rounds +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +%macro AESENC_ROUNDS_x16 5 +%define %%L00_03 %1 ; [in/out] ZMM with lane 0-3 blocks +%define %%L04_07 %2 ; [in/out] ZMM with lane 4-7 blocks +%define %%L08_11 %3 ; [in/out] ZMM with lane 8-11 blocks +%define %%L12_15 %4 ; [in/out] ZMM with lane 12-15 blocks +%define %%NROUNDS %5 ; [in] number of aes rounds + +%define %%KP ARG + _aesarg_key_tab +%define K00_03_OFFSET 0 +%define K04_07_OFFSET 64 +%define K08_11_OFFSET 128 +%define K12_15_OFFSET 192 + +%assign ROUND 0 +%rep (%%NROUNDS + 2) + +%if ROUND < 1 + ;; XOR with key 0 before doing aesenc + vpxorq %%L00_03, [%%KP + K00_03_OFFSET + ROUND * (16*16)] + vpxorq %%L04_07, [%%KP + K04_07_OFFSET + ROUND * (16*16)] + vpxorq %%L08_11, [%%KP + K08_11_OFFSET + ROUND * (16*16)] + vpxorq %%L12_15, [%%KP + K12_15_OFFSET + ROUND * (16*16)] +%else +%if ROUND <= %%NROUNDS + + ;; rounds 1 to 9/11/13 + vaesenc %%L00_03, %%L00_03, [%%KP + K00_03_OFFSET + ROUND * (16*16)] + vaesenc %%L04_07, %%L04_07, [%%KP + K04_07_OFFSET + ROUND * (16*16)] + vaesenc %%L08_11, %%L08_11, [%%KP + K08_11_OFFSET + ROUND * (16*16)] + vaesenc %%L12_15, %%L12_15, [%%KP + K12_15_OFFSET + ROUND * (16*16)] +%else + ;; the last round + vaesenclast %%L00_03, %%L00_03, [%%KP + K00_03_OFFSET + ROUND * (16*16)] + vaesenclast %%L04_07, %%L04_07, [%%KP + K04_07_OFFSET + ROUND * (16*16)] + vaesenclast %%L08_11, %%L08_11, [%%KP + K08_11_OFFSET + ROUND * (16*16)] + vaesenclast %%L12_15, %%L12_15, [%%KP + K12_15_OFFSET + ROUND * (16*16)] +%endif +%endif + +%assign ROUND (ROUND + 1) +%endrep + +%endmacro ; AESENC_ROUNDS_x16 + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; ENCRYPT_16_PARALLEL - Encode all blocks up to multiple of 4 +; - Operation +; - loop encrypting %%LENGTH bytes of input data +; - each loop encrypts 4 blocks across 16 lanes +; - stop when %%LENGTH is less than 64 bytes (4 blocks) +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +%macro ENCRYPT_16_PARALLEL 31 +%define %%ZIV00_03 %1 ;; [in] lane 0-3 IVs +%define %%ZIV04_07 %2 ;; [in] lane 4-7 IVs +%define %%ZIV08_11 %3 ;; [in] lane 8-11 IVs +%define %%ZIV12_15 %4 ;; [in] lane 12-15 IVs +%define %%LENGTH %5 ;; [in/out] GP register with length in bytes +%define %%NROUNDS %6 ;; [in] Number of AES rounds; numerical value +%define %%IDX %7 ;; [clobbered] GP reg to maintain idx +%define %%B0L00_03 %8 ;; [clobbered] tmp ZMM register +%define %%B0L04_07 %9 ;; [clobbered] tmp ZMM register +%define %%B0L08_11 %10 ;; [clobbered] tmp ZMM register +%define %%B0L12_15 %11 ;; [clobbered] tmp ZMM register +%define %%B1L00_03 %12 ;; [clobbered] tmp ZMM register +%define %%B1L04_07 %13 ;; [clobbered] tmp ZMM register +%define %%B1L08_11 %14 ;; [clobbered] tmp ZMM register +%define %%B1L12_15 %15 ;; [clobbered] tmp ZMM register +%define %%B2L00_03 %16 ;; [clobbered] tmp ZMM register +%define %%B2L04_07 %17 ;; [clobbered] tmp ZMM register +%define %%B2L08_11 %18 ;; [clobbered] tmp ZMM register +%define %%B2L12_15 %19 ;; [clobbered] tmp ZMM register +%define %%B3L00_03 %20 ;; [clobbered] tmp ZMM register +%define %%B3L04_07 %21 ;; [clobbered] tmp ZMM register +%define %%B3L08_11 %22 ;; [clobbered] tmp ZMM register +%define %%B3L12_15 %23 ;; [clobbered] tmp ZMM register +%define %%ZTMP0 %24 ;; [clobbered] tmp ZMM register +%define %%ZTMP1 %25 ;; [clobbered] tmp ZMM register +%define %%ZTMP2 %26 ;; [clobbered] tmp ZMM register +%define %%ZTMP3 %27 ;; [clobbered] tmp ZMM register +%define %%TMP0 %28 ;; [clobbered] tmp GP register +%define %%TMP1 %29 ;; [clobbered] tmp GP register +%define %%TMP2 %30 ;; [clobbered] tmp GP register +%define %%TMP3 %31 ;; [clobbered] tmp GP register + +%define %%IN ARG + _aesarg_in +%define %%OUT ARG + _aesarg_out + + ;; check for at least 4 blocks + cmp %%LENGTH, 64 + jl %%encrypt_16_done + + xor %%IDX, %%IDX + ;; skip length check on first loop + jmp %%encrypt_16_first + +%%encrypt_16_start: + cmp %%LENGTH, 64 + jl %%encrypt_16_end + +%%encrypt_16_first: + ;; load 4 plaintext blocks for lanes 0-3 + LOAD_STORE_x4 0, 1, 2, 3, %%IN, %%IDX, %%B0L00_03, %%B1L00_03, \ + %%B2L00_03, %%B3L00_03, %%TMP0, %%TMP1, %%TMP2, %%TMP3, LOAD + + TRANSPOSE_4x4 %%B0L00_03, %%B1L00_03, %%B2L00_03, %%B3L00_03, \ + %%ZTMP0, %%ZTMP1, %%ZTMP2, %%ZTMP3 + + ;; load 4 plaintext blocks for lanes 4-7 + LOAD_STORE_x4 4, 5, 6, 7, %%IN, %%IDX, %%B0L04_07, %%B1L04_07, \ + %%B2L04_07, %%B3L04_07, %%TMP0, %%TMP1, %%TMP2, %%TMP3, LOAD + + TRANSPOSE_4x4 %%B0L04_07, %%B1L04_07, %%B2L04_07, %%B3L04_07, \ + %%ZTMP0, %%ZTMP1, %%ZTMP2, %%ZTMP3 + + ;; load 4 plaintext blocks for lanes 8-11 + LOAD_STORE_x4 8, 9, 10, 11, %%IN, %%IDX, %%B0L08_11, %%B1L08_11, \ + %%B2L08_11, %%B3L08_11, %%TMP0, %%TMP1, %%TMP2, %%TMP3, LOAD + + TRANSPOSE_4x4 %%B0L08_11, %%B1L08_11, %%B2L08_11, %%B3L08_11, \ + %%ZTMP0, %%ZTMP1, %%ZTMP2, %%ZTMP3 + + ;; load 4 plaintext blocks for lanes 12-15 + LOAD_STORE_x4 12, 13, 14, 15, %%IN, %%IDX, %%B0L12_15, %%B1L12_15, \ + %%B2L12_15, %%B3L12_15, %%TMP0, %%TMP1, %%TMP2, %%TMP3, LOAD + + TRANSPOSE_4x4 %%B0L12_15, %%B1L12_15, %%B2L12_15, %%B3L12_15, \ + %%ZTMP0, %%ZTMP1, %%ZTMP2, %%ZTMP3 + + ;; xor first plaintext block with IV + vpxorq %%B0L00_03, %%ZIV00_03 + vpxorq %%B0L04_07, %%ZIV04_07 + vpxorq %%B0L08_11, %%ZIV08_11 + vpxorq %%B0L12_15, %%ZIV12_15 + + ;; encrypt block 0 lanes + AESENC_ROUNDS_x16 %%B0L00_03, %%B0L04_07, %%B0L08_11, %%B0L12_15, %%NROUNDS + + ;; xor plaintext block with last cipher block + vpxorq %%B1L00_03, %%B0L00_03 + vpxorq %%B1L04_07, %%B0L04_07 + vpxorq %%B1L08_11, %%B0L08_11 + vpxorq %%B1L12_15, %%B0L12_15 + + ;; encrypt block 1 lanes + AESENC_ROUNDS_x16 %%B1L00_03, %%B1L04_07, %%B1L08_11, %%B1L12_15, %%NROUNDS + + ;; xor plaintext block with last cipher block + vpxorq %%B2L00_03, %%B1L00_03 + vpxorq %%B2L04_07, %%B1L04_07 + vpxorq %%B2L08_11, %%B1L08_11 + vpxorq %%B2L12_15, %%B1L12_15 + + ;; encrypt block 2 lanes + AESENC_ROUNDS_x16 %%B2L00_03, %%B2L04_07, %%B2L08_11, %%B2L12_15, %%NROUNDS + + ;; xor plaintext block with last cipher block + vpxorq %%B3L00_03, %%B2L00_03 + vpxorq %%B3L04_07, %%B2L04_07 + vpxorq %%B3L08_11, %%B2L08_11 + vpxorq %%B3L12_15, %%B2L12_15 + + ;; encrypt block 3 lanes + AESENC_ROUNDS_x16 %%B3L00_03, %%B3L04_07, %%B3L08_11, %%B3L12_15, %%NROUNDS + + ;; store last cipher block + vmovdqa64 %%ZIV00_03, %%B3L00_03 + vmovdqa64 %%ZIV04_07, %%B3L04_07 + vmovdqa64 %%ZIV08_11, %%B3L08_11 + vmovdqa64 %%ZIV12_15, %%B3L12_15 + + ;; write back cipher text for lanes 0-3 + TRANSPOSE_4x4 %%B0L00_03, %%B1L00_03, %%B2L00_03, %%B3L00_03, \ + %%ZTMP0, %%ZTMP1, %%ZTMP2, %%ZTMP3 + + LOAD_STORE_x4 0, 1, 2, 3, %%OUT, %%IDX, %%B0L00_03, %%B1L00_03, \ + %%B2L00_03, %%B3L00_03, %%TMP0, %%TMP1, %%TMP2, %%TMP3, STORE + + ;; write back cipher text for lanes 4-7 + TRANSPOSE_4x4 %%B0L04_07, %%B1L04_07, %%B2L04_07, %%B3L04_07, \ + %%ZTMP0, %%ZTMP1, %%ZTMP2, %%ZTMP3 + + LOAD_STORE_x4 4, 5, 6, 7, %%OUT, %%IDX, %%B0L04_07, %%B1L04_07, \ + %%B2L04_07, %%B3L04_07, %%TMP0, %%TMP1, %%TMP2, %%TMP3, STORE + + ;; write back cipher text for lanes 8-11 + TRANSPOSE_4x4 %%B0L08_11, %%B1L08_11, %%B2L08_11, %%B3L08_11, \ + %%ZTMP0, %%ZTMP1, %%ZTMP2, %%ZTMP3 + + LOAD_STORE_x4 8, 9, 10, 11, %%OUT, %%IDX, %%B0L08_11, %%B1L08_11, \ + %%B2L08_11, %%B3L08_11, %%TMP0, %%TMP1, %%TMP2, %%TMP3, STORE + + ;; write back cipher text for lanes 12-15 + TRANSPOSE_4x4 %%B0L12_15, %%B1L12_15, %%B2L12_15, %%B3L12_15, \ + %%ZTMP0, %%ZTMP1, %%ZTMP2, %%ZTMP3 + + LOAD_STORE_x4 12, 13, 14, 15, %%OUT, %%IDX, %%B0L12_15, %%B1L12_15, \ + %%B2L12_15, %%B3L12_15, %%TMP0, %%TMP1, %%TMP2, %%TMP3, STORE + + sub %%LENGTH, 64 + add %%IDX, 64 + jmp %%encrypt_16_start + +%%encrypt_16_end: + ;; update in/out pointers + vpbroadcastq %%ZTMP2, %%IDX + vpaddq %%ZTMP0, %%ZTMP2, [%%IN] + vpaddq %%ZTMP1, %%ZTMP2, [%%IN + 64] + vmovdqa64 [%%IN], %%ZTMP0 + vmovdqa64 [%%IN + 64], %%ZTMP1 + + vpaddq %%ZTMP0, %%ZTMP2, [%%OUT] + vpaddq %%ZTMP1, %%ZTMP2, [%%OUT + 64] + vmovdqa64 [%%OUT], %%ZTMP0 + vmovdqa64 [%%OUT + 64], %%ZTMP1 + +%%encrypt_16_done: +%endmacro + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; ENCRYPT_16_FINAL Encodes final blocks (less than 4) across 16 lanes +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +%macro ENCRYPT_16_FINAL 31 +%define %%ZIV00_03 %1 ;; [in] lane 0-3 IVs +%define %%ZIV04_07 %2 ;; [in] lane 4-7 IVs +%define %%ZIV08_11 %3 ;; [in] lane 8-11 IVs +%define %%ZIV12_15 %4 ;; [in] lane 12-15 IVs +%define %%NROUNDS %5 ;; [in] Number of AES rounds; numerical value +%define %%IDX %6 ;; [clobbered] GP reg to maintain idx +%define %%B0L00_03 %7 ;; [clobbered] tmp ZMM register +%define %%B0L04_07 %8 ;; [clobbered] tmp ZMM register +%define %%B0L08_11 %9 ;; [clobbered] tmp ZMM register +%define %%B0L12_15 %10 ;; [clobbered] tmp ZMM register +%define %%B1L00_03 %11 ;; [clobbered] tmp ZMM register +%define %%B1L04_07 %12 ;; [clobbered] tmp ZMM register +%define %%B1L08_11 %13 ;; [clobbered] tmp ZMM register +%define %%B1L12_15 %14 ;; [clobbered] tmp ZMM register +%define %%B2L00_03 %15 ;; [clobbered] tmp ZMM register +%define %%B2L04_07 %16 ;; [clobbered] tmp ZMM register +%define %%B2L08_11 %17 ;; [clobbered] tmp ZMM register +%define %%B2L12_15 %18 ;; [clobbered] tmp ZMM register +%define %%B3L00_03 %19 ;; [clobbered] tmp ZMM register +%define %%B3L04_07 %20 ;; [clobbered] tmp ZMM register +%define %%B3L08_11 %21 ;; [clobbered] tmp ZMM register +%define %%B3L12_15 %22 ;; [clobbered] tmp ZMM register +%define %%ZTMP0 %23 ;; [clobbered] tmp ZMM register +%define %%ZTMP1 %24 ;; [clobbered] tmp ZMM register +%define %%ZTMP2 %25 ;; [clobbered] tmp ZMM register +%define %%ZTMP3 %26 ;; [clobbered] tmp ZMM register +%define %%TMP0 %27 ;; [clobbered] tmp GP register +%define %%TMP1 %28 ;; [clobbered] tmp GP register +%define %%TMP2 %29 ;; [clobbered] tmp GP register +%define %%TMP3 %30 ;; [clobbered] tmp GP register +%define %%NUM_BLKS %31 ;; [in] number of blocks (numerical value) + +%define %%IN ARG + _aesarg_in +%define %%OUT ARG + _aesarg_out + +%if %%NUM_BLKS == 1 + mov %%TMP0, 0x0000_0000_0000_ffff + kmovq k1, %%TMP0 +%elif %%NUM_BLKS == 2 + mov %%TMP0, 0x0000_0000_ffff_ffff + kmovq k1, %%TMP0 +%elif %%NUM_BLKS == 3 + mov %%TMP0, 0x0000_ffff_ffff_ffff + kmovq k1, %%TMP0 +%endif + xor %%IDX, %%IDX + + ;; load 4 plaintext blocks for lanes 0-3 + LOAD_STORE_x4 0, 1, 2, 3, %%IN, %%IDX, %%B0L00_03, %%B1L00_03, \ + %%B2L00_03, %%B3L00_03, %%TMP0, %%TMP1, %%TMP2, \ + %%TMP3, LOAD, k1 + + TRANSPOSE_4x4 %%B0L00_03, %%B1L00_03, %%B2L00_03, %%B3L00_03, \ + %%ZTMP0, %%ZTMP1, %%ZTMP2, %%ZTMP3 + + ;; load 4 plaintext blocks for lanes 4-7 + LOAD_STORE_x4 4, 5, 6, 7, %%IN, %%IDX, %%B0L04_07, %%B1L04_07, \ + %%B2L04_07, %%B3L04_07, %%TMP0, %%TMP1, %%TMP2, \ + %%TMP3, LOAD, k1 + + TRANSPOSE_4x4 %%B0L04_07, %%B1L04_07, %%B2L04_07, %%B3L04_07, \ + %%ZTMP0, %%ZTMP1, %%ZTMP2, %%ZTMP3 + + ;; load 4 plaintext blocks for lanes 8-11 + LOAD_STORE_x4 8, 9, 10, 11, %%IN, %%IDX, %%B0L08_11, %%B1L08_11, \ + %%B2L08_11, %%B3L08_11, %%TMP0, %%TMP1, %%TMP2, \ + %%TMP3, LOAD, k1 + + TRANSPOSE_4x4 %%B0L08_11, %%B1L08_11, %%B2L08_11, %%B3L08_11, \ + %%ZTMP0, %%ZTMP1, %%ZTMP2, %%ZTMP3 + + ;; load 4 plaintext blocks for lanes 12-15 + LOAD_STORE_x4 12, 13, 14, 15, %%IN, %%IDX, %%B0L12_15, %%B1L12_15, \ + %%B2L12_15, %%B3L12_15, %%TMP0, %%TMP1, %%TMP2, \ + %%TMP3, LOAD, k1 + + TRANSPOSE_4x4 %%B0L12_15, %%B1L12_15, %%B2L12_15, %%B3L12_15, \ + %%ZTMP0, %%ZTMP1, %%ZTMP2, %%ZTMP3 + + ;; xor plaintext block with IV + vpxorq %%B0L00_03, %%ZIV00_03 + vpxorq %%B0L04_07, %%ZIV04_07 + vpxorq %%B0L08_11, %%ZIV08_11 + vpxorq %%B0L12_15, %%ZIV12_15 + + ;; encrypt block 0 lanes + AESENC_ROUNDS_x16 %%B0L00_03, %%B0L04_07, %%B0L08_11, %%B0L12_15, %%NROUNDS + +%if %%NUM_BLKS == 1 + ;; store last cipher block + vmovdqa64 %%ZIV00_03, %%B0L00_03 + vmovdqa64 %%ZIV04_07, %%B0L04_07 + vmovdqa64 %%ZIV08_11, %%B0L08_11 + vmovdqa64 %%ZIV12_15, %%B0L12_15 +%endif + +%if %%NUM_BLKS >= 2 + ;; xor plaintext block with last cipher block + vpxorq %%B1L00_03, %%B0L00_03 + vpxorq %%B1L04_07, %%B0L04_07 + vpxorq %%B1L08_11, %%B0L08_11 + vpxorq %%B1L12_15, %%B0L12_15 + + ;; encrypt block 1 lanes + AESENC_ROUNDS_x16 %%B1L00_03, %%B1L04_07, %%B1L08_11, %%B1L12_15, %%NROUNDS +%endif +%if %%NUM_BLKS == 2 + ;; store last cipher block + vmovdqa64 %%ZIV00_03, %%B1L00_03 + vmovdqa64 %%ZIV04_07, %%B1L04_07 + vmovdqa64 %%ZIV08_11, %%B1L08_11 + vmovdqa64 %%ZIV12_15, %%B1L12_15 +%endif + +%if %%NUM_BLKS >= 3 + ;; xor plaintext block with last cipher block + vpxorq %%B2L00_03, %%B1L00_03 + vpxorq %%B2L04_07, %%B1L04_07 + vpxorq %%B2L08_11, %%B1L08_11 + vpxorq %%B2L12_15, %%B1L12_15 + + ;; encrypt block 2 lanes + AESENC_ROUNDS_x16 %%B2L00_03, %%B2L04_07, %%B2L08_11, %%B2L12_15, %%NROUNDS +%endif +%if %%NUM_BLKS == 3 + ;; store last cipher block + vmovdqa64 %%ZIV00_03, %%B2L00_03 + vmovdqa64 %%ZIV04_07, %%B2L04_07 + vmovdqa64 %%ZIV08_11, %%B2L08_11 + vmovdqa64 %%ZIV12_15, %%B2L12_15 +%endif + ;; write back cipher text for lanes 0-3 + TRANSPOSE_4x4 %%B0L00_03, %%B1L00_03, %%B2L00_03, %%B3L00_03, \ + %%ZTMP0, %%ZTMP1, %%ZTMP2, %%ZTMP3 + + LOAD_STORE_x4 0, 1, 2, 3, %%OUT, %%IDX, %%B0L00_03, %%B1L00_03, \ + %%B2L00_03, %%B3L00_03, %%TMP0, %%TMP1, %%TMP2, \ + %%TMP3, STORE, k1 + + ;; write back cipher text for lanes 4-7 + TRANSPOSE_4x4 %%B0L04_07, %%B1L04_07, %%B2L04_07, %%B3L04_07, \ + %%ZTMP0, %%ZTMP1, %%ZTMP2, %%ZTMP3 + + LOAD_STORE_x4 4, 5, 6, 7, %%OUT, %%IDX, %%B0L04_07, %%B1L04_07, \ + %%B2L04_07, %%B3L04_07, %%TMP0, %%TMP1, %%TMP2, \ + %%TMP3, STORE, k1 + + ;; write back cipher text for lanes 8-11 + TRANSPOSE_4x4 %%B0L08_11, %%B1L08_11, %%B2L08_11, %%B3L08_11, \ + %%ZTMP0, %%ZTMP1, %%ZTMP2, %%ZTMP3 + + LOAD_STORE_x4 8, 9, 10, 11, %%OUT, %%IDX, %%B0L08_11, %%B1L08_11, \ + %%B2L08_11, %%B3L08_11, %%TMP0, %%TMP1, %%TMP2, \ + %%TMP3, STORE, k1 + + ;; write back cipher text for lanes 12-15 + TRANSPOSE_4x4 %%B0L12_15, %%B1L12_15, %%B2L12_15, %%B3L12_15, \ + %%ZTMP0, %%ZTMP1, %%ZTMP2, %%ZTMP3 + + LOAD_STORE_x4 12, 13, 14, 15, %%OUT, %%IDX, %%B0L12_15, %%B1L12_15, \ + %%B2L12_15, %%B3L12_15, %%TMP0, %%TMP1, %%TMP2, \ + %%TMP3, STORE, k1 + + ;; update in/out pointers + mov %%IDX, %%NUM_BLKS + shl %%IDX, 4 + vpbroadcastq %%ZTMP2, %%IDX + vpaddq %%ZTMP0, %%ZTMP2, [%%IN] + vpaddq %%ZTMP1, %%ZTMP2, [%%IN + 64] + vmovdqa64 [%%IN], %%ZTMP0 + vmovdqa64 [%%IN + 64], %%ZTMP1 + + vpaddq %%ZTMP0, %%ZTMP2, [%%OUT] + vpaddq %%ZTMP1, %%ZTMP2, [%%OUT + 64] + vmovdqa64 [%%OUT], %%ZTMP0 + vmovdqa64 [%%OUT + 64], %%ZTMP1 + +%endmacro + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; CBC_ENC Encodes given data. +; Requires the input data be at least 1 block (16 bytes) long +; Input: Number of AES rounds +; +; First encrypts block up to multiple of 4 +; Then encrypts final blocks (less than 4) +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +%macro CBC_ENC 1 +%define %%ROUNDS %1 + + ;; load transpose tables + vmovdqa64 TAB_A0B0A1B1, [rel A0B0A1B1] + vmovdqa64 TAB_A2B2A3B3, [rel A2B2A3B3] + + ;; load IV's per lane + vmovdqa64 ZIV00_03, [ARG + _aesarg_IV + 16*0] + vmovdqa64 ZIV04_07, [ARG + _aesarg_IV + 16*4] + vmovdqa64 ZIV08_11, [ARG + _aesarg_IV + 16*8] + vmovdqa64 ZIV12_15, [ARG + _aesarg_IV + 16*12] + + ENCRYPT_16_PARALLEL ZIV00_03, ZIV04_07, ZIV08_11, ZIV12_15, \ + LEN, %%ROUNDS, IA12, ZT0, ZT1, ZT2, ZT3, ZT4, ZT5, \ + ZT6, ZT7, ZT8, ZT9, ZT10, ZT11, ZT12, ZT13, ZT14, \ + ZT15, ZT16, ZT17, ZT18, ZT19, IA2, IA3, IA4, IA5 + + ;; get num remaining blocks + shr LEN, 4 + and LEN, 3 + je %%_cbc_enc_done + cmp LEN, 1 + je %%_final_blocks_1 + cmp LEN, 2 + je %%_final_blocks_2 + +%%_final_blocks_3: + ENCRYPT_16_FINAL ZIV00_03, ZIV04_07, ZIV08_11, ZIV12_15, \ + %%ROUNDS, IA12, ZT0, ZT1, ZT2, ZT3, ZT4, ZT5, ZT6, ZT7, \ + ZT8, ZT9, ZT10, ZT11, ZT12, ZT13, ZT14, ZT15, ZT16, ZT17, \ + ZT18, ZT19, IA2, IA3, IA4, IA5, 3 + jmp %%_cbc_enc_done +%%_final_blocks_1: + ENCRYPT_16_FINAL ZIV00_03, ZIV04_07, ZIV08_11, ZIV12_15, \ + %%ROUNDS, IA12, ZT0, ZT1, ZT2, ZT3, ZT4, ZT5, ZT6, ZT7, \ + ZT8, ZT9, ZT10, ZT11, ZT12, ZT13, ZT14, ZT15, ZT16, ZT17, \ + ZT18, ZT19, IA2, IA3, IA4, IA5, 1 + jmp %%_cbc_enc_done +%%_final_blocks_2: + ENCRYPT_16_FINAL ZIV00_03, ZIV04_07, ZIV08_11, ZIV12_15, \ + %%ROUNDS, IA12, ZT0, ZT1, ZT2, ZT3, ZT4, ZT5, ZT6, ZT7, \ + ZT8, ZT9, ZT10, ZT11, ZT12, ZT13, ZT14, ZT15, ZT16, ZT17, \ + ZT18, ZT19, IA2, IA3, IA4, IA5, 2 +%%_cbc_enc_done: + ;; store IV's per lane + vmovdqa64 [ARG + _aesarg_IV + 16*0], ZIV00_03 + vmovdqa64 [ARG + _aesarg_IV + 16*4], ZIV04_07 + vmovdqa64 [ARG + _aesarg_IV + 16*8], ZIV08_11 + vmovdqa64 [ARG + _aesarg_IV + 16*12], ZIV12_15 +%endmacro + + +section .data +;;;;;;;;;;;;;;;;;; +; Transpose tables +;;;;;;;;;;;;;;;;;; +default rel + +align 64 +A0B0A1B1: + dq 0x0, 0x1, 0x8, 0x9, 0x2, 0x3, 0xa, 0xb + +align 64 +A2B2A3B3: + dq 0x4, 0x5, 0xc, 0xd, 0x6, 0x7, 0xe, 0xf + + +section .text + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; void aes_cbc_enc_128_vaes_avx512(AES_ARGS *args, uint64_t len_in_bytes); +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +MKGLOBAL(aes_cbc_enc_128_vaes_avx512,function,internal) +aes_cbc_enc_128_vaes_avx512: + FUNC_SAVE + CBC_ENC 9 + FUNC_RESTORE + ret + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; void aes_cbc_enc_192_vaes_avx512(AES_ARGS *args, uint64_t len_in_bytes); +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +MKGLOBAL(aes_cbc_enc_192_vaes_avx512,function,internal) +aes_cbc_enc_192_vaes_avx512: + FUNC_SAVE + CBC_ENC 11 + FUNC_RESTORE + ret +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; void aes_cbc_enc_256_vaes_avx512(AES_ARGS *args, uint64_t len_in_bytes); +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +MKGLOBAL(aes_cbc_enc_256_vaes_avx512,function,internal) +aes_cbc_enc_256_vaes_avx512: + FUNC_SAVE + CBC_ENC 13 + FUNC_RESTORE + ret + +%ifdef LINUX +section .note.GNU-stack noalloc noexec nowrite progbits +%endif diff --git a/src/spdk/intel-ipsec-mb/avx512/cntr_vaes_avx512.asm b/src/spdk/intel-ipsec-mb/avx512/cntr_vaes_avx512.asm new file mode 100644 index 000000000..50ff86b6e --- /dev/null +++ b/src/spdk/intel-ipsec-mb/avx512/cntr_vaes_avx512.asm @@ -0,0 +1,1524 @@ +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; Copyright(c) 2019, Intel Corporation All rights reserved. +; +; Redistribution and use in source and binary forms, with or without +; modification, are permitted provided that the following conditions +; are met: +; * Redistributions of source code must retain the above copyright +; notice, this list of conditions and the following disclaimer. +; * Redistributions in binary form must reproduce the above copyright +; notice, this list of conditions and the following disclaimer in +; the documentation and/or other materials provided with the +; distribution. +; * Neither the name of Intel Corporation nor the names of its +; contributors may be used to endorse or promote products derived +; from this software without specific prior written permission. +; +; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +; "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +; LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +; A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +; OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +; SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +; LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +; DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +; THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +%include "include/os.asm" +%include "include/reg_sizes.asm" +%include "mb_mgr_datastruct.asm" +%include "job_aes_hmac.asm" +%include "include/memcpy.asm" + +%include "include/aes_common.asm" +%include "include/const.inc" + +section .data +default rel + +align 16 +ONE: + dq 0x0000000000000001, 0x0000000000000000 + +align 64 +SHUF_MASK: + dq 0x08090A0B0C0D0E0F, 0x0001020304050607 + dq 0x08090A0B0C0D0E0F, 0x0001020304050607 + dq 0x08090A0B0C0D0E0F, 0x0001020304050607 + dq 0x08090A0B0C0D0E0F, 0x0001020304050607 + +align 64 +ddq_add_13_16: + dq 0x000000000000000d, 0x0000000000000000 + dq 0x000000000000000e, 0x0000000000000000 + dq 0x000000000000000f, 0x0000000000000000 + dq 0x0000000000000010, 0x0000000000000000 + +align 64 +ddq_add_9_12: + dq 0x0000000000000009, 0x0000000000000000 + dq 0x000000000000000a, 0x0000000000000000 + dq 0x000000000000000b, 0x0000000000000000 + dq 0x000000000000000c, 0x0000000000000000 + +align 64 +ddq_add_5_8: + dq 0x0000000000000005, 0x0000000000000000 + dq 0x0000000000000006, 0x0000000000000000 + dq 0x0000000000000007, 0x0000000000000000 + dq 0x0000000000000008, 0x0000000000000000 + +align 64 +ddq_add_1_4: + dq 0x0000000000000001, 0x0000000000000000 + dq 0x0000000000000002, 0x0000000000000000 + dq 0x0000000000000003, 0x0000000000000000 + dq 0x0000000000000004, 0x0000000000000000 + +align 64 +ddq_add_12_15: + dq 0x000000000000000c, 0x0000000000000000 + dq 0x000000000000000d, 0x0000000000000000 + dq 0x000000000000000e, 0x0000000000000000 + dq 0x000000000000000f, 0x0000000000000000 + +align 64 +ddq_add_8_11: + dq 0x0000000000000008, 0x0000000000000000 + dq 0x0000000000000009, 0x0000000000000000 + dq 0x000000000000000a, 0x0000000000000000 + dq 0x000000000000000b, 0x0000000000000000 + +align 64 +ddq_add_4_7: + dq 0x0000000000000004, 0x0000000000000000 + dq 0x0000000000000005, 0x0000000000000000 + dq 0x0000000000000006, 0x0000000000000000 + dq 0x0000000000000007, 0x0000000000000000 + +align 64 +ddq_add_0_3: + dq 0x0000000000000000, 0x0000000000000000 + dq 0x0000000000000001, 0x0000000000000000 + dq 0x0000000000000002, 0x0000000000000000 + dq 0x0000000000000003, 0x0000000000000000 + +align 64 +ddq_add_16: + dq 0x0000000000000010, 0x0000000000000000 + dq 0x0000000000000010, 0x0000000000000000 + dq 0x0000000000000010, 0x0000000000000000 + dq 0x0000000000000010, 0x0000000000000000 + +align 64 +byte64_len_to_mask_table: + dq 0x0000000000000000, 0x0000000000000001 + dq 0x0000000000000003, 0x0000000000000007 + dq 0x000000000000000f, 0x000000000000001f + dq 0x000000000000003f, 0x000000000000007f + dq 0x00000000000000ff, 0x00000000000001ff + dq 0x00000000000003ff, 0x00000000000007ff + dq 0x0000000000000fff, 0x0000000000001fff + dq 0x0000000000003fff, 0x0000000000007fff + dq 0x000000000000ffff, 0x000000000001ffff + dq 0x000000000003ffff, 0x000000000007ffff + dq 0x00000000000fffff, 0x00000000001fffff + dq 0x00000000003fffff, 0x00000000007fffff + dq 0x0000000000ffffff, 0x0000000001ffffff + dq 0x0000000003ffffff, 0x0000000007ffffff + dq 0x000000000fffffff, 0x000000001fffffff + dq 0x000000003fffffff, 0x000000007fffffff + dq 0x00000000ffffffff, 0x00000001ffffffff + dq 0x00000003ffffffff, 0x00000007ffffffff + dq 0x0000000fffffffff, 0x0000001fffffffff + dq 0x0000003fffffffff, 0x0000007fffffffff + dq 0x000000ffffffffff, 0x000001ffffffffff + dq 0x000003ffffffffff, 0x000007ffffffffff + dq 0x00000fffffffffff, 0x00001fffffffffff + dq 0x00003fffffffffff, 0x00007fffffffffff + dq 0x0000ffffffffffff, 0x0001ffffffffffff + dq 0x0003ffffffffffff, 0x0007ffffffffffff + dq 0x000fffffffffffff, 0x001fffffffffffff + dq 0x003fffffffffffff, 0x007fffffffffffff + dq 0x00ffffffffffffff, 0x01ffffffffffffff + dq 0x03ffffffffffffff, 0x07ffffffffffffff + dq 0x0fffffffffffffff, 0x1fffffffffffffff + dq 0x3fffffffffffffff, 0x7fffffffffffffff + dq 0xffffffffffffffff + +align 16 +initial_12_IV_counter: + dq 0x0000000000000000, 0x0100000000000000 + +mask_16_bytes: + dq 0x000000000000ffff + +section .text +default rel + +%ifdef LINUX +%define arg1 rdi +%else +%define arg1 rcx +%endif + +%define ZKEY0 zmm17 +%define ZKEY1 zmm18 +%define ZKEY2 zmm19 +%define ZKEY3 zmm20 +%define ZKEY4 zmm21 +%define ZKEY5 zmm22 +%define ZKEY6 zmm23 +%define ZKEY7 zmm24 +%define ZKEY8 zmm25 +%define ZKEY9 zmm26 +%define ZKEY10 zmm27 +%define ZKEY11 zmm28 +%define ZKEY12 zmm29 +%define ZKEY13 zmm30 +%define ZKEY14 zmm31 + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;; Stack frame definition +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +%ifidn __OUTPUT_FORMAT__, win64 + %define GP_STORAGE (7*8) ; space for 7 GP registers +%else + %define GP_STORAGE (5*8) ; space for 5 GP registers +%endif + +%define STACK_FRAME_SIZE GP_STORAGE + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;; Utility Macros +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;; This macro is used to maintain the bits from the output text +;;; when writing out the output blocks, in case there are some bits +;;; that do not require encryption +%macro PRESERVE_BITS 12-13 +%define %%RBITS %1 ; [in] Remaining bits in last byte +%define %%LENGTH %2 ; [in] Length of the last set of blocks +%define %%CYPH_PLAIN_OUT %3 ; [in] Pointer to output buffer +%define %%ZIN_OUT %4 ; [in/out] ZMM with last set of output blocks +%define %%ZTMP0 %5 ; [clobbered] ZMM temporary +%define %%ZTMP1 %6 ; [clobbered] ZMM temporary +%define %%ZTMP2 %7 ; [clobbered] ZMM temporary +%define %%IA0 %8 ; [clobbered] GP temporary +%define %%IA1 %9 ; [clobbered] GP temporary +%define %%blocks_to_skip %10 ; [in] Number of blocks to skip from output +%define %%FULL_PARTIAL %11 ; [in] Last block type selection "full" or "partial" +%define %%MASKREG %12 ; [clobbered] Mask register +%define %%DATA_OFFSET %13 ; [in/out] Data offset +%define %%NUM_ARGS %0 + +;; offset = number of sets of 4 blocks to skip +%assign offset (((%%blocks_to_skip) / 4) * 64) +;; num_left_blocks = number of blocks in the last set +%assign num_left_blocks (((%%blocks_to_skip) & 3) + 1) ;; Range 1-4 blocks + +%if %%NUM_ARGS == 13 + ;; Load output to get last partial byte +%ifidn %%FULL_PARTIAL, partial + vmovdqu8 %%ZTMP0{%%MASKREG}, [%%CYPH_PLAIN_OUT + %%DATA_OFFSET + offset] +%else + vmovdqu8 %%ZTMP0, [%%CYPH_PLAIN_OUT + %%DATA_OFFSET + offset] +%endif ; %%FULL_PARTIAL == partial +%else + ;; Load output to get last partial byte (loading up to the last 4 blocks) + ZMM_LOAD_MASKED_BLOCKS_0_16 num_left_blocks, %%CYPH_PLAIN_OUT, offset, \ + %%ZTMP0, no_zmm, no_zmm, no_zmm, %%MASKREG +%endif ;; %%NUM_ARGS == 13 + + ;; Save RCX in temporary GP register + mov %%IA0, rcx + mov DWORD(%%IA1), 0xff + mov cl, BYTE(%%RBITS) + shr DWORD(%%IA1), cl ;; e.g. 3 remaining bits -> mask = 00011111 + mov rcx, %%IA0 + + vmovq XWORD(%%ZTMP1), %%IA1 + + ;; Get number of full bytes in last block. + ;; Subtracting the bytes in the blocks to skip to the length of whole + ;; set of blocks gives us the number of bytes in the last block, + ;; but the last block has a partial byte at the end, so an extra byte + ;; needs to be subtracted + mov %%IA1, %%LENGTH + sub %%IA1, (%%blocks_to_skip * 16 + 1) + XVPSLLB XWORD(%%ZTMP1), %%IA1, XWORD(%%ZTMP2), %%IA0 +%if num_left_blocks == 4 + vshufi64x2 %%ZTMP1, %%ZTMP1, %%ZTMP1, 0x15 +%elif num_left_blocks == 3 + vshufi64x2 %%ZTMP1, %%ZTMP1, %%ZTMP1, 0x45 +%elif num_left_blocks == 2 + vshufi64x2 %%ZTMP1, %%ZTMP1, %%ZTMP1, 0x51 +%endif ;; No need to shift if there is only one block + + ;; At this point, ZTMP1 contains a mask with all 0s, but with some ones + ;; in the partial byte + + ;; First, clear the last bits (not to be ciphered) of the last output block + ;; %%ZIN_OUT = %%ZIN_OUT AND NOT %%ZTMP1 (0x50 = andA!C) + vpternlogq %%ZIN_OUT, %%ZTMP1, %%ZTMP1, 0x50 + + ;; Then, set these last bits to the last bits coming from the output + ;; %%ZIN_OUT = %%ZIN_OUT OR (%%ZTMP0 AND %%ZTMP1) (0xF8 = orAandBC) + vpternlogq %%ZIN_OUT, %%ZTMP0, %%ZTMP1, 0xF8 + +%endmacro +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;; This macro is used to "warm-up" pipeline for ENCRYPT_16_PARALLEL +;;; macro code. It is called only for data lengths 256 and above. +;;; The flow is as follows: +;;; - encrypt the initial %%num_initial_blocks blocks (can be 0) +;;; - encrypt the next 16 blocks +;;; - the last 16th block can be partial (lengths between 257 and 367) +;;; - partial block ciphering is handled within this macro + +%macro INITIAL_BLOCKS 26 +%define %%KEY %1 ; [in] pointer to key +%define %%CYPH_PLAIN_OUT %2 ; [in] output buffer +%define %%PLAIN_CYPH_IN %3 ; [in] input buffer +%define %%LENGTH %4 ; [in/out] number of bytes to process +%define %%DATA_OFFSET %5 ; [in/out] data offset +%define %%num_initial_blocks %6 ; [in] can be between 0 and 15 +%define %%CTR %7 ; [in] XMM first counter block +%define %%CTR_1_4 %8 ; [out] ZMM next 1-4 counter blocks +%define %%CTR_5_8 %9 ; [out] ZMM next 5-8 counter blocks +%define %%CTR_9_12 %10 ; [out] ZMM next 9-12 counter blocks +%define %%CTR_13_16 %11 ; [out] ZMM next 13-16 counter blocks +%define %%ZT1 %12 ; [clobbered] ZMM temporary +%define %%ZT2 %13 ; [clobbered] ZMM temporary +%define %%ZT3 %14 ; [clobbered] ZMM temporary +%define %%ZT4 %15 ; [clobbered] ZMM temporary +%define %%ZT5 %16 ; [clobbered] ZMM temporary +%define %%ZT6 %17 ; [clobbered] ZMM temporary +%define %%ZT7 %18 ; [clobbered] ZMM temporary +%define %%ZT8 %19 ; [clobbered] ZMM temporary +%define %%IA0 %20 ; [clobbered] GP temporary +%define %%IA1 %21 ; [clobbered] GP temporary +%define %%MASKREG %22 ; [clobbered] mask register +%define %%SHUFREG %23 ; [in] ZMM register with shuffle mask +%define %%NROUNDS %24 ; [in] number of rounds; numerical value +%define %%CNTR_TYPE %25 ; [in] Type of CNTR operation to do (CNTR/CNTR_BIT) +%define %%RBITS %26 ; [in] Number of remaining bits in last byte + +%define %%T1 XWORD(%%ZT1) +%define %%T2 XWORD(%%ZT2) +%define %%T3 XWORD(%%ZT3) +%define %%T4 XWORD(%%ZT4) +%define %%T5 XWORD(%%ZT5) +%define %%T6 XWORD(%%ZT6) +%define %%T7 XWORD(%%ZT7) +%define %%T8 XWORD(%%ZT8) + +%ifidn %%CNTR_TYPE, CNTR +%define %%VPADD vpaddd +%else +%define %%VPADD vpaddq +%endif + +%if %%num_initial_blocks > 0 + ;; load plain/cipher text + ZMM_LOAD_BLOCKS_0_16 %%num_initial_blocks, %%PLAIN_CYPH_IN, 0, \ + %%ZT5, %%ZT6, %%ZT7, %%ZT8, load_4_instead_of_3 + + ;; prepare AES counter blocks +%if %%num_initial_blocks > 1 +%if %%num_initial_blocks == 2 + vshufi64x2 YWORD(%%ZT1), YWORD(%%CTR), YWORD(%%CTR), 0 + %%VPADD YWORD(%%ZT1), YWORD(%%ZT1), [rel ddq_add_0_3] +%elif %%num_initial_blocks <= 4 + vshufi64x2 ZWORD(%%CTR), ZWORD(%%CTR), ZWORD(%%CTR), 0 + %%VPADD %%ZT1, ZWORD(%%CTR), [rel ddq_add_0_3] +%elif %%num_initial_blocks <= 8 + vshufi64x2 ZWORD(%%CTR), ZWORD(%%CTR), ZWORD(%%CTR), 0 + %%VPADD %%ZT1, ZWORD(%%CTR), [rel ddq_add_0_3] + %%VPADD %%ZT2, ZWORD(%%CTR), [rel ddq_add_4_7] +%elif %%num_initial_blocks <= 12 + vshufi64x2 ZWORD(%%CTR), ZWORD(%%CTR), ZWORD(%%CTR), 0 + %%VPADD %%ZT1, ZWORD(%%CTR), [rel ddq_add_0_3] + %%VPADD %%ZT2, ZWORD(%%CTR), [rel ddq_add_4_7] + %%VPADD %%ZT3, ZWORD(%%CTR), [rel ddq_add_8_11] +%else + vshufi64x2 ZWORD(%%CTR), ZWORD(%%CTR), ZWORD(%%CTR), 0 + %%VPADD %%ZT1, ZWORD(%%CTR), [rel ddq_add_0_3] + %%VPADD %%ZT2, ZWORD(%%CTR), [rel ddq_add_4_7] + %%VPADD %%ZT3, ZWORD(%%CTR), [rel ddq_add_8_11] + %%VPADD %%ZT4, ZWORD(%%CTR), [rel ddq_add_12_15] +%endif +%endif + + ;; extract new counter value (%%T1) + ;; shuffle the counters for AES rounds +%if %%num_initial_blocks == 1 + vpshufb %%T1, %%CTR, XWORD(%%SHUFREG) +%elif %%num_initial_blocks == 2 + vextracti32x4 %%CTR, YWORD(%%ZT1), 1 + vpshufb YWORD(%%ZT1), YWORD(%%SHUFREG) +%elif %%num_initial_blocks <= 4 + vextracti32x4 %%CTR, %%ZT1, (%%num_initial_blocks - 1) + vpshufb %%ZT1, %%SHUFREG +%elif %%num_initial_blocks == 5 + vmovdqa64 %%CTR, %%T2 + vpshufb %%ZT1, %%SHUFREG + vpshufb %%T2, XWORD(%%SHUFREG) +%elif %%num_initial_blocks == 6 + vextracti32x4 %%CTR, YWORD(%%ZT2), 1 + vpshufb %%ZT1, %%SHUFREG + vpshufb YWORD(%%ZT2), YWORD(%%SHUFREG) +%elif %%num_initial_blocks = 7 + vextracti32x4 %%CTR, %%ZT2, 2 + vpshufb %%ZT1, %%SHUFREG + vpshufb %%ZT2, %%SHUFREG +%elif %%num_initial_blocks = 8 + vextracti32x4 %%CTR, %%ZT2, 3 + vpshufb %%ZT1, %%SHUFREG + vpshufb %%ZT2, %%SHUFREG +%elif %%num_initial_blocks = 9 + vmovdqa64 %%CTR, %%T3 + vpshufb %%ZT1, %%SHUFREG + vpshufb %%ZT2, %%SHUFREG + vpshufb %%T3, XWORD(%%SHUFREG) +%elif %%num_initial_blocks = 10 + vextracti32x4 %%CTR, YWORD(%%ZT3), 1 + vpshufb %%ZT1, %%SHUFREG + vpshufb %%ZT2, %%SHUFREG + vpshufb YWORD(%%ZT3), YWORD(%%SHUFREG) +%elif %%num_initial_blocks = 11 + vextracti32x4 %%CTR, %%ZT3, 2 + vpshufb %%ZT1, %%SHUFREG + vpshufb %%ZT2, %%SHUFREG + vpshufb %%ZT3, %%SHUFREG +%elif %%num_initial_blocks = 12 + vextracti32x4 %%CTR, %%ZT3, 3 + vpshufb %%ZT1, %%SHUFREG + vpshufb %%ZT2, %%SHUFREG + vpshufb %%ZT3, %%SHUFREG +%elif %%num_initial_blocks = 13 + vmovdqa64 %%CTR, %%T4 + vpshufb %%ZT1, %%SHUFREG + vpshufb %%ZT2, %%SHUFREG + vpshufb %%ZT3, %%SHUFREG + vpshufb %%T4, XWORD(%%SHUFREG) +%elif %%num_initial_blocks = 14 + vextracti32x4 %%CTR, YWORD(%%ZT4), 1 + vpshufb %%ZT1, %%SHUFREG + vpshufb %%ZT2, %%SHUFREG + vpshufb %%ZT3, %%SHUFREG + vpshufb YWORD(%%ZT4), YWORD(%%SHUFREG) +%elif %%num_initial_blocks = 15 + vextracti32x4 %%CTR, %%ZT4, 2 + vpshufb %%ZT1, %%SHUFREG + vpshufb %%ZT2, %%SHUFREG + vpshufb %%ZT3, %%SHUFREG + vpshufb %%ZT4, %%SHUFREG +%endif + + ;; AES rounds and XOR with plain/cipher text +%assign j 0 +%rep (%%NROUNDS + 2) + ZMM_AESENC_ROUND_BLOCKS_0_16 \ + %%ZT1, %%ZT2, %%ZT3, %%ZT4, ZKEY %+ j, j, \ + %%ZT5, %%ZT6, %%ZT7, %%ZT8, %%num_initial_blocks, \ + %%NROUNDS +%assign j (j + 1) +%endrep + + ;; write cipher/plain text back to output + ZMM_STORE_BLOCKS_0_16 %%num_initial_blocks, %%CYPH_PLAIN_OUT, 0, \ + %%ZT1, %%ZT2, %%ZT3, %%ZT4 + + ;; adjust data offset and length + sub %%LENGTH, (%%num_initial_blocks * 16) + add %%DATA_OFFSET, (%%num_initial_blocks * 16) +%endif ; %%num_initial_blocks > 0 + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; - cipher of %%num_initial_blocks is done + ;; - prepare counter blocks for the next 16 blocks (ZT5-ZT8) + ;; - shuffle the blocks for AES + ;; - encrypt the next 16 blocks + + ;; get text load/store mask (assume full mask by default) + mov %%IA0, 0xffff_ffff_ffff_ffff +%if %%num_initial_blocks > 0 + ;; NOTE: 'jge' is always taken for %%num_initial_blocks = 0 + ;; This macro is executed for length 256 and up, + ;; zero length is checked in CNTR_ENC_DEC. + ;; We know there is partial block if: + ;; LENGTH - 16*num_initial_blocks < 256 + cmp %%LENGTH, 256 + jge %%_initial_partial_block_continue + mov %%IA1, rcx + mov rcx, 256 + sub rcx, %%LENGTH + shr %%IA0, cl + mov rcx, %%IA1 +%%_initial_partial_block_continue: +%endif + kmovq %%MASKREG, %%IA0 + ;; load plain or cipher text + vmovdqu8 %%ZT5, [%%PLAIN_CYPH_IN + %%DATA_OFFSET] + vmovdqu8 %%ZT6, [%%PLAIN_CYPH_IN + %%DATA_OFFSET + 64] + vmovdqu8 %%ZT7, [%%PLAIN_CYPH_IN + %%DATA_OFFSET + 128] + vmovdqu8 %%ZT8{%%MASKREG}{z}, [%%PLAIN_CYPH_IN + %%DATA_OFFSET + 192] + + ;; prepare next counter blocks + vshufi64x2 ZWORD(%%CTR), ZWORD(%%CTR), ZWORD(%%CTR), 0 +%if %%num_initial_blocks > 0 + vpaddd %%CTR_1_4, ZWORD(%%CTR), [rel ddq_add_1_4] + vpaddd %%CTR_5_8, ZWORD(%%CTR), [rel ddq_add_5_8] + vpaddd %%CTR_9_12, ZWORD(%%CTR), [rel ddq_add_9_12] + vpaddd %%CTR_13_16, ZWORD(%%CTR), [rel ddq_add_13_16] +%else + vpaddd %%CTR_1_4, ZWORD(%%CTR), [rel ddq_add_0_3] + vpaddd %%CTR_5_8, ZWORD(%%CTR), [rel ddq_add_4_7] + vpaddd %%CTR_9_12, ZWORD(%%CTR), [rel ddq_add_8_11] + vpaddd %%CTR_13_16, ZWORD(%%CTR), [rel ddq_add_12_15] +%endif + + vpshufb %%ZT1, %%CTR_1_4, %%SHUFREG + vpshufb %%ZT2, %%CTR_5_8, %%SHUFREG + vpshufb %%ZT3, %%CTR_9_12, %%SHUFREG + vpshufb %%ZT4, %%CTR_13_16, %%SHUFREG + + ;; AES rounds and XOR with plain/cipher text +%assign j 0 +%rep (%%NROUNDS + 2) + ZMM_AESENC_ROUND_BLOCKS_0_16 \ + %%ZT1, %%ZT2, %%ZT3, %%ZT4, ZKEY %+ j, j, \ + %%ZT5, %%ZT6, %%ZT7, %%ZT8, 16, %%NROUNDS +%assign j (j + 1) +%endrep + +%ifidn %%CNTR_TYPE, CNTR_BIT + ;; check if this is the end of the message + cmp %%LENGTH, 256 + jg %%store_output + ;; Check if there is a partial byte + or %%RBITS, %%RBITS + jz %%store_output + + ;; Copy the bits that are not ciphered from the output text, + ;; into the last bits of the output block, before writing it out + PRESERVE_BITS %%RBITS, %%LENGTH, %%CYPH_PLAIN_OUT, %%ZT4, %%ZT5, %%ZT6, %%ZT7, \ + %%IA0, %%IA1, 15, partial, %%MASKREG, %%DATA_OFFSET + +%endif + +%%store_output: + ;; write cipher/plain text back to output + vmovdqu8 [%%CYPH_PLAIN_OUT + %%DATA_OFFSET], %%ZT1 + vmovdqu8 [%%CYPH_PLAIN_OUT + %%DATA_OFFSET + 64], %%ZT2 + vmovdqu8 [%%CYPH_PLAIN_OUT + %%DATA_OFFSET + 128], %%ZT3 + vmovdqu8 [%%CYPH_PLAIN_OUT + %%DATA_OFFSET + 192]{%%MASKREG}, %%ZT4 + + ;; check if there is partial block + cmp %%LENGTH, 256 + jl %%_initial_partial_done + ;; adjust offset and length + add %%DATA_OFFSET, 256 + sub %%LENGTH, 256 + jmp %%_initial_blocks_done +%%_initial_partial_done: + ;; zero the length (all encryption is complete) + xor %%LENGTH, %%LENGTH +%%_initial_blocks_done: + +%endmacro ; INITIAL_BLOCKS +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;; INITIAL_BLOCKS_PARTIAL macro with support for a partial final block. +;;; It may look similar to INITIAL_BLOCKS but its usage is different: +;;; - It is not meant to cipher counter blocks for the main by16 loop. +;;; Just ciphers amount of blocks. +;;; - Small packets (<256 bytes) +;;; +;;; num_initial_blocks is expected to include the partial final block +;;; in the count. +%macro INITIAL_BLOCKS_PARTIAL 21 +%define %%KEY %1 ; [in] key pointer +%define %%CYPH_PLAIN_OUT %2 ; [in] text out pointer +%define %%PLAIN_CYPH_IN %3 ; [in] text out pointer +%define %%LENGTH %4 ; [in/clobbered] length in bytes +%define %%num_initial_blocks %5 ; [in] can be from 1 to 16 (not 0) +%define %%CTR %6 ; [in/out] current counter value +%define %%ZT1 %7 ; [clobbered] ZMM temporary +%define %%ZT2 %8 ; [clobbered] ZMM temporary +%define %%ZT3 %9 ; [clobbered] ZMM temporary +%define %%ZT4 %10 ; [clobbered] ZMM temporary +%define %%ZT5 %11 ; [clobbered] ZMM temporary +%define %%ZT6 %12 ; [clobbered] ZMM temporary +%define %%ZT7 %13 ; [clobbered] ZMM temporary +%define %%ZT8 %14 ; [clobbered] ZMM temporary +%define %%IA0 %15 ; [clobbered] GP temporary +%define %%IA1 %16 ; [clobbered] GP temporary +%define %%MASKREG %17 ; [clobbered] mask register +%define %%SHUFREG %18 ; [in] ZMM register with shuffle mask +%define %%NROUNDS %19 ; [in] number of rounds; numerical value +%define %%CNTR_TYPE %20 ; [in] Type of CNTR operation to do (CNTR/CNTR_BIT) +%define %%RBITS %21 ; [in] Number of remaining bits in last byte + +%define %%T1 XWORD(%%ZT1) +%define %%T2 XWORD(%%ZT2) +%define %%T3 XWORD(%%ZT3) +%define %%T4 XWORD(%%ZT4) +%define %%T5 XWORD(%%ZT5) +%define %%T6 XWORD(%%ZT6) +%define %%T7 XWORD(%%ZT7) +%define %%T8 XWORD(%%ZT8) + + ;; get load/store mask + lea %%IA0, [rel byte64_len_to_mask_table] + mov %%IA1, %%LENGTH +%if %%num_initial_blocks > 12 + sub %%IA1, 192 +%elif %%num_initial_blocks > 8 + sub %%IA1, 128 +%elif %%num_initial_blocks > 4 + sub %%IA1, 64 +%endif + kmovq %%MASKREG, [%%IA0 + %%IA1*8] + + ;; load plain/cipher text + ZMM_LOAD_MASKED_BLOCKS_0_16 %%num_initial_blocks, %%PLAIN_CYPH_IN, 0, \ + %%ZT5, %%ZT6, %%ZT7, %%ZT8, %%MASKREG + + ;; prepare AES counter blocks +%if %%num_initial_blocks == 1 + vmovdqa64 XWORD(%%ZT1), XWORD(%%CTR) +%elif %%num_initial_blocks == 2 + vshufi64x2 YWORD(%%ZT1), YWORD(%%CTR), YWORD(%%CTR), 0 + vpaddd YWORD(%%ZT1), YWORD(%%ZT1), [rel ddq_add_0_3] +%elif %%num_initial_blocks <= 4 + vshufi64x2 ZWORD(%%CTR), ZWORD(%%CTR), ZWORD(%%CTR), 0 + vpaddd %%ZT1, ZWORD(%%CTR), [rel ddq_add_0_3] +%elif %%num_initial_blocks <= 8 + vshufi64x2 ZWORD(%%CTR), ZWORD(%%CTR), ZWORD(%%CTR), 0 + vpaddd %%ZT1, ZWORD(%%CTR), [rel ddq_add_0_3] + vpaddd %%ZT2, ZWORD(%%CTR), [rel ddq_add_4_7] +%elif %%num_initial_blocks <= 12 + vshufi64x2 ZWORD(%%CTR), ZWORD(%%CTR), ZWORD(%%CTR), 0 + vpaddd %%ZT1, ZWORD(%%CTR), [rel ddq_add_0_3] + vpaddd %%ZT2, ZWORD(%%CTR), [rel ddq_add_4_7] + vpaddd %%ZT3, ZWORD(%%CTR), [rel ddq_add_8_11] +%else + vshufi64x2 ZWORD(%%CTR), ZWORD(%%CTR), ZWORD(%%CTR), 0 + vpaddd %%ZT1, ZWORD(%%CTR), [rel ddq_add_0_3] + vpaddd %%ZT2, ZWORD(%%CTR), [rel ddq_add_4_7] + vpaddd %%ZT3, ZWORD(%%CTR), [rel ddq_add_8_11] + vpaddd %%ZT4, ZWORD(%%CTR), [rel ddq_add_12_15] +%endif + + ;; shuffle the counters for AES rounds + ZMM_OPCODE3_DSTR_SRC1R_SRC2R_BLOCKS_0_16 %%num_initial_blocks, vpshufb, \ + %%ZT1, %%ZT2, %%ZT3, %%ZT4, \ + %%ZT1, %%ZT2, %%ZT3, %%ZT4, \ + %%SHUFREG, %%SHUFREG, %%SHUFREG, %%SHUFREG + + ;; AES rounds and XOR with plain/cipher text +%assign j 0 +%rep (%%NROUNDS + 2) + ZMM_AESENC_ROUND_BLOCKS_0_16 \ + %%ZT1, %%ZT2, %%ZT3, %%ZT4, ZKEY %+ j, j, \ + %%ZT5, %%ZT6, %%ZT7, %%ZT8, %%num_initial_blocks, \ + %%NROUNDS +%assign j (j + 1) +%endrep + +%ifidn %%CNTR_TYPE, CNTR_BIT + ;; Check if there is a partial byte + or %%RBITS, %%RBITS + jz %%store_output + + ;; Copy the bits that are not ciphered from the output text, + ;; into the last bits of the output block, before writing it out +%if %%num_initial_blocks <= 4 + PRESERVE_BITS %%RBITS, %%LENGTH, %%CYPH_PLAIN_OUT, %%ZT1, %%ZT5, %%ZT6, %%ZT7, \ + %%IA0, %%IA1, (%%num_initial_blocks - 1), \ + partial, %%MASKREG +%elif %%num_initial_blocks <= 8 + PRESERVE_BITS %%RBITS, %%LENGTH, %%CYPH_PLAIN_OUT, %%ZT2, %%ZT5, %%ZT6, %%ZT7, \ + %%IA0, %%IA1, (%%num_initial_blocks - 1), \ + partial, %%MASKREG +%elif %%num_initial_blocks <= 12 + PRESERVE_BITS %%RBITS, %%LENGTH, %%CYPH_PLAIN_OUT, %%ZT3, %%ZT5, %%ZT6, %%ZT7, \ + %%IA0, %%IA1, (%%num_initial_blocks - 1), \ + partial, %%MASKREG +%else + PRESERVE_BITS %%RBITS, %%LENGTH, %%CYPH_PLAIN_OUT, %%ZT4, %%ZT5, %%ZT6, %%ZT7, \ + %%IA0, %%IA1, (%%num_initial_blocks - 1), \ + partial, %%MASKREG +%endif + +%endif + +%%store_output: + ;; write cipher/plain text back to output + ZMM_STORE_MASKED_BLOCKS_0_16 %%num_initial_blocks, %%CYPH_PLAIN_OUT, 0, \ + %%ZT1, %%ZT2, %%ZT3, %%ZT4, %%MASKREG + +%endmacro ; INITIAL_BLOCKS_PARTIAL + + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;; Main CNTR macro +;;; - operates on single stream +;;; - encrypts 16 blocks at a time +%macro ENCRYPT_16_PARALLEL 26 +%define %%KEY %1 ; [in] key pointer +%define %%CYPH_PLAIN_OUT %2 ; [in] pointer to output buffer +%define %%PLAIN_CYPH_IN %3 ; [in] pointer to input buffer +%define %%DATA_OFFSET %4 ; [in] data offset +%define %%CTR_1_4 %5 ; [in/out] ZMM next 1-4 counter blocks +%define %%CTR_5_8 %6 ; [in/out] ZMM next 5-8 counter blocks +%define %%CTR_9_12 %7 ; [in/out] ZMM next 9-12 counter blocks +%define %%CTR_13_16 %8 ; [in/out] ZMM next 13-16 counter blocks +%define %%FULL_PARTIAL %9 ; [in] last block type selection "full" or "partial" +%define %%IA0 %10 ; [clobbered] temporary GP register +%define %%IA1 %11 ; [clobbered] temporary GP register +%define %%LENGTH %12 ; [in] length +%define %%ZT1 %13 ; [clobbered] temporary ZMM (cipher) +%define %%ZT2 %14 ; [clobbered] temporary ZMM (cipher) +%define %%ZT3 %15 ; [clobbered] temporary ZMM (cipher) +%define %%ZT4 %16 ; [clobbered] temporary ZMM (cipher) +%define %%ZT5 %17 ; [clobbered] temporary ZMM (cipher) +%define %%ZT6 %18 ; [clobbered] temporary ZMM (cipher) +%define %%ZT7 %19 ; [clobbered] temporary ZMM (cipher) +%define %%ZT8 %20 ; [clobbered] temporary ZMM (cipher) +%define %%MASKREG %21 ; [clobbered] mask register for partial loads/stores +%define %%SHUFREG %22 ; [in] ZMM register with shuffle mask +%define %%ADD8REG %23 ; [in] ZMM register with increment by 8 mask +%define %%NROUNDS %24 ; [in] number of rounds; numerical value +%define %%CNTR_TYPE %25 ; [in] Type of CNTR operation to do (CNTR/CNTR_BIT) +%define %%RBITS %26 ; [in] Number of remaining bits in last byte + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; load/store mask (partial case) and load the text data +%ifidn %%FULL_PARTIAL, full + vmovdqu8 %%ZT5, [%%PLAIN_CYPH_IN + %%DATA_OFFSET] + vmovdqu8 %%ZT6, [%%PLAIN_CYPH_IN + %%DATA_OFFSET + 64] + vmovdqu8 %%ZT7, [%%PLAIN_CYPH_IN + %%DATA_OFFSET + 128] + vmovdqu8 %%ZT8, [%%PLAIN_CYPH_IN + %%DATA_OFFSET + 192] +%else + lea %%IA0, [rel byte64_len_to_mask_table] + mov %%IA1, %%LENGTH + sub %%IA1, (3*64) + kmovq %%MASKREG, [%%IA0 + 8*%%IA1] + vmovdqu8 %%ZT5, [%%PLAIN_CYPH_IN + %%DATA_OFFSET] + vmovdqu8 %%ZT6, [%%PLAIN_CYPH_IN + %%DATA_OFFSET + 64] + vmovdqu8 %%ZT7, [%%PLAIN_CYPH_IN + %%DATA_OFFSET + 128] + vmovdqu8 %%ZT8{%%MASKREG}{z}, [%%PLAIN_CYPH_IN + %%DATA_OFFSET + 192] +%endif + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; populate counter blocks + ;; %%CTR is shuffled outside the scope of this macro + ;; it has to be kept in unshuffled form + vpaddd %%CTR_1_4, %%CTR_1_4, %%ADD8REG + vpaddd %%CTR_5_8, %%CTR_5_8, %%ADD8REG + vpaddd %%CTR_9_12, %%CTR_9_12, %%ADD8REG + vpaddd %%CTR_13_16, %%CTR_13_16, %%ADD8REG + vpshufb %%ZT1, %%CTR_1_4, %%SHUFREG + vpshufb %%ZT2, %%CTR_5_8, %%SHUFREG + vpshufb %%ZT3, %%CTR_9_12, %%SHUFREG + vpshufb %%ZT4, %%CTR_13_16, %%SHUFREG + +%assign j 0 +%rep (%%NROUNDS + 2) + ZMM_AESENC_ROUND_BLOCKS_0_16 \ + %%ZT1, %%ZT2, %%ZT3, %%ZT4, ZKEY %+ j, j, \ + %%ZT5, %%ZT6, %%ZT7, %%ZT8, 16, %%NROUNDS +%assign j (j + 1) +%endrep + +%ifidn %%CNTR_TYPE, CNTR_BIT + ;; Check if this is the last round + cmp %%LENGTH, 256 + jg %%store_output + ;; Check if there is a partial byte + or %%RBITS, %%RBITS + jz %%store_output + + ;; Copy the bits that are not ciphered from the output text, + ;; into the last bits of the output block, before writing it out + PRESERVE_BITS %%RBITS, %%LENGTH, %%CYPH_PLAIN_OUT, %%ZT4, %%ZT5, %%ZT6, %%ZT7, \ + %%IA0, %%IA1, 15, %%FULL_PARTIAL, %%MASKREG, %%DATA_OFFSET + +%endif + +%%store_output: + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; store the text data +%ifidn %%FULL_PARTIAL, full + vmovdqu8 [%%CYPH_PLAIN_OUT + %%DATA_OFFSET], %%ZT1 + vmovdqu8 [%%CYPH_PLAIN_OUT + %%DATA_OFFSET + 64], %%ZT2 + vmovdqu8 [%%CYPH_PLAIN_OUT + %%DATA_OFFSET + 128], %%ZT3 + vmovdqu8 [%%CYPH_PLAIN_OUT + %%DATA_OFFSET + 192], %%ZT4 +%else + vmovdqu8 [%%CYPH_PLAIN_OUT + %%DATA_OFFSET], %%ZT1 + vmovdqu8 [%%CYPH_PLAIN_OUT + %%DATA_OFFSET + 64], %%ZT2 + vmovdqu8 [%%CYPH_PLAIN_OUT + %%DATA_OFFSET + 128], %%ZT3 + vmovdqu8 [%%CYPH_PLAIN_OUT + %%DATA_OFFSET + 192]{%%MASKREG}, %%ZT4 +%endif + +%endmacro ; ENCRYPT_16_PARALLEL + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;; Save register content for the caller +%macro FUNC_SAVE 1 +%define %%CNTR_TYPE %1 ; [in] Type of CNTR operation to do (CNTR/CNTR_BIT) + mov rax, rsp + + sub rsp, STACK_FRAME_SIZE + and rsp, ~63 + + mov [rsp + 0*8], r12 + mov [rsp + 1*8], r13 +%ifidn %%CNTR_TYPE, CNTR_BIT + mov [rsp + 2*8], r14 +%endif + mov [rsp + 3*8], rax ; stack +%ifidn __OUTPUT_FORMAT__, win64 + mov [rsp + 4*8], rdi + mov [rsp + 5*8], rsi +%endif + +%endmacro + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;; Restore register content for the caller +%macro FUNC_RESTORE 1 +%define %%CNTR_TYPE %1 ; [in] Type of CNTR operation to do (CNTR/CNTR_BIT) + + vzeroupper +%ifidn __OUTPUT_FORMAT__, win64 + mov rdi, [rsp + 4*8] + mov rsi, [rsp + 5*8] +%endif + mov r12, [rsp + 0*8] + mov r13, [rsp + 1*8] +%ifidn %%CNTR_TYPE, CNTR_BIT + mov r14, [rsp + 2*8] +%endif + mov rsp, [rsp + 3*8] ; stack +%endmacro + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;; Cipher payloads shorter than 256 bytes +;;; - number of blocks in the message comes as argument +;;; - depending on the number of blocks an optimized variant of +;;; INITIAL_BLOCKS_PARTIAL is invoked +%macro CNTR_ENC_DEC_SMALL 21 +%define %%KEY %1 ; [in] key pointer +%define %%CYPH_PLAIN_OUT %2 ; [in] output buffer +%define %%PLAIN_CYPH_IN %3 ; [in] input buffer +%define %%LENGTH %4 ; [in] data length +%define %%NUM_BLOCKS %5 ; [in] number of blocks to process 1 to 8 +%define %%CTR %6 ; [in/out] XMM counter block +%define %%ZTMP1 %7 ; [clobbered] ZMM register +%define %%ZTMP2 %8 ; [clobbered] ZMM register +%define %%ZTMP3 %9 ; [clobbered] ZMM register +%define %%ZTMP4 %10 ; [clobbered] ZMM register +%define %%ZTMP5 %11 ; [clobbered] ZMM register +%define %%ZTMP6 %12 ; [clobbered] ZMM register +%define %%ZTMP7 %13 ; [clobbered] ZMM register +%define %%ZTMP8 %14 ; [clobbered] ZMM register +%define %%IA0 %15 ; [clobbered] GP register +%define %%IA1 %16 ; [clobbered] GP register +%define %%MASKREG %17 ; [clobbered] mask register +%define %%SHUFREG %18 ; [in] ZMM register with shuffle mask +%define %%NROUNDS %19 ; [in] number of rounds; numerical value +%define %%CNTR_TYPE %20 ; [in] Type of CNTR operation to do (CNTR/CNTR_BIT) +%define %%RBITS %21 ; [in] Number of remaining bits in last byte + + cmp %%NUM_BLOCKS, 8 + je %%_small_initial_num_blocks_is_8 + jl %%_small_initial_blocks_is_1_7 + + ; Initial blocks 9-16 + cmp %%NUM_BLOCKS, 12 + je %%_small_initial_num_blocks_is_12 + jl %%_small_initial_blocks_is_9_11 + + ; Initial blocks 13-16 + cmp %%NUM_BLOCKS, 16 + je %%_small_initial_num_blocks_is_16 + cmp %%NUM_BLOCKS, 15 + je %%_small_initial_num_blocks_is_15 + cmp %%NUM_BLOCKS, 14 + je %%_small_initial_num_blocks_is_14 + cmp %%NUM_BLOCKS, 13 + je %%_small_initial_num_blocks_is_13 + +%%_small_initial_blocks_is_9_11: + cmp %%NUM_BLOCKS, 11 + je %%_small_initial_num_blocks_is_11 + cmp %%NUM_BLOCKS, 10 + je %%_small_initial_num_blocks_is_10 + cmp %%NUM_BLOCKS, 9 + je %%_small_initial_num_blocks_is_9 + +%%_small_initial_blocks_is_1_7: + cmp %%NUM_BLOCKS, 4 + je %%_small_initial_num_blocks_is_4 + jl %%_small_initial_blocks_is_1_3 + + ; Initial blocks 5-7 + cmp %%NUM_BLOCKS, 7 + je %%_small_initial_num_blocks_is_7 + cmp %%NUM_BLOCKS, 6 + je %%_small_initial_num_blocks_is_6 + cmp %%NUM_BLOCKS, 5 + je %%_small_initial_num_blocks_is_5 + +%%_small_initial_blocks_is_1_3: + cmp %%NUM_BLOCKS, 3 + je %%_small_initial_num_blocks_is_3 + cmp %%NUM_BLOCKS, 2 + je %%_small_initial_num_blocks_is_2 + + jmp %%_small_initial_num_blocks_is_1 + + +%%_small_initial_num_blocks_is_16: + INITIAL_BLOCKS_PARTIAL %%KEY, %%CYPH_PLAIN_OUT, \ + %%PLAIN_CYPH_IN, %%LENGTH, 16, \ + %%CTR, \ + %%ZTMP1, %%ZTMP2, %%ZTMP3, %%ZTMP4, %%ZTMP5, \ + %%ZTMP6, %%ZTMP7, %%ZTMP8, \ + %%IA0, %%IA1, %%MASKREG, %%SHUFREG, %%NROUNDS, \ + %%CNTR_TYPE, %%RBITS + jmp %%_small_initial_blocks_encrypted + +%%_small_initial_num_blocks_is_15: + INITIAL_BLOCKS_PARTIAL %%KEY, %%CYPH_PLAIN_OUT, \ + %%PLAIN_CYPH_IN, %%LENGTH, 15, \ + %%CTR, \ + %%ZTMP1, %%ZTMP2, %%ZTMP3, %%ZTMP4, %%ZTMP5, \ + %%ZTMP6, %%ZTMP7, %%ZTMP8, \ + %%IA0, %%IA1, %%MASKREG, %%SHUFREG, %%NROUNDS, \ + %%CNTR_TYPE, %%RBITS + jmp %%_small_initial_blocks_encrypted + +%%_small_initial_num_blocks_is_14: + INITIAL_BLOCKS_PARTIAL %%KEY, %%CYPH_PLAIN_OUT, \ + %%PLAIN_CYPH_IN, %%LENGTH, 14, \ + %%CTR, \ + %%ZTMP1, %%ZTMP2, %%ZTMP3, %%ZTMP4, %%ZTMP5, \ + %%ZTMP6, %%ZTMP7, %%ZTMP8, \ + %%IA0, %%IA1, %%MASKREG, %%SHUFREG, %%NROUNDS, \ + %%CNTR_TYPE, %%RBITS + jmp %%_small_initial_blocks_encrypted + +%%_small_initial_num_blocks_is_13: + INITIAL_BLOCKS_PARTIAL %%KEY, %%CYPH_PLAIN_OUT, \ + %%PLAIN_CYPH_IN, %%LENGTH, 13, \ + %%CTR, \ + %%ZTMP1, %%ZTMP2, %%ZTMP3, %%ZTMP4, %%ZTMP5, \ + %%ZTMP6, %%ZTMP7, %%ZTMP8, \ + %%IA0, %%IA1, %%MASKREG, %%SHUFREG, %%NROUNDS, \ + %%CNTR_TYPE, %%RBITS + jmp %%_small_initial_blocks_encrypted + +%%_small_initial_num_blocks_is_12: + INITIAL_BLOCKS_PARTIAL %%KEY, %%CYPH_PLAIN_OUT, \ + %%PLAIN_CYPH_IN, %%LENGTH, 12, \ + %%CTR, \ + %%ZTMP1, %%ZTMP2, %%ZTMP3, %%ZTMP4, %%ZTMP5, \ + %%ZTMP6, %%ZTMP7, %%ZTMP8, \ + %%IA0, %%IA1, %%MASKREG, %%SHUFREG, %%NROUNDS, \ + %%CNTR_TYPE, %%RBITS + jmp %%_small_initial_blocks_encrypted + +%%_small_initial_num_blocks_is_11: + INITIAL_BLOCKS_PARTIAL %%KEY, %%CYPH_PLAIN_OUT, \ + %%PLAIN_CYPH_IN, %%LENGTH, 11, \ + %%CTR, \ + %%ZTMP1, %%ZTMP2, %%ZTMP3, %%ZTMP4, %%ZTMP5, \ + %%ZTMP6, %%ZTMP7, %%ZTMP8, \ + %%IA0, %%IA1, %%MASKREG, %%SHUFREG, %%NROUNDS, \ + %%CNTR_TYPE, %%RBITS + jmp %%_small_initial_blocks_encrypted + +%%_small_initial_num_blocks_is_10: + INITIAL_BLOCKS_PARTIAL %%KEY, %%CYPH_PLAIN_OUT, \ + %%PLAIN_CYPH_IN, %%LENGTH, 10, \ + %%CTR, \ + %%ZTMP1, %%ZTMP2, %%ZTMP3, %%ZTMP4, %%ZTMP5, \ + %%ZTMP6, %%ZTMP7, %%ZTMP8, \ + %%IA0, %%IA1, %%MASKREG, %%SHUFREG, %%NROUNDS, \ + %%CNTR_TYPE, %%RBITS + jmp %%_small_initial_blocks_encrypted + +%%_small_initial_num_blocks_is_9: + INITIAL_BLOCKS_PARTIAL %%KEY, %%CYPH_PLAIN_OUT, \ + %%PLAIN_CYPH_IN, %%LENGTH, 9, \ + %%CTR, \ + %%ZTMP1, %%ZTMP2, %%ZTMP3, %%ZTMP4, %%ZTMP5, \ + %%ZTMP6, %%ZTMP7, %%ZTMP8, \ + %%IA0, %%IA1, %%MASKREG, %%SHUFREG, %%NROUNDS, \ + %%CNTR_TYPE, %%RBITS + jmp %%_small_initial_blocks_encrypted +%%_small_initial_num_blocks_is_8: + INITIAL_BLOCKS_PARTIAL %%KEY, %%CYPH_PLAIN_OUT, \ + %%PLAIN_CYPH_IN, %%LENGTH, 8, \ + %%CTR, \ + %%ZTMP1, %%ZTMP2, %%ZTMP3, %%ZTMP4, %%ZTMP5, \ + %%ZTMP6, %%ZTMP7, %%ZTMP8, \ + %%IA0, %%IA1, %%MASKREG, %%SHUFREG, %%NROUNDS, \ + %%CNTR_TYPE, %%RBITS + jmp %%_small_initial_blocks_encrypted + +%%_small_initial_num_blocks_is_7: + INITIAL_BLOCKS_PARTIAL %%KEY, %%CYPH_PLAIN_OUT, \ + %%PLAIN_CYPH_IN, %%LENGTH, 7, \ + %%CTR, \ + %%ZTMP1, %%ZTMP2, %%ZTMP3, %%ZTMP4, %%ZTMP5, \ + %%ZTMP6, %%ZTMP7, %%ZTMP8, \ + %%IA0, %%IA1, %%MASKREG, %%SHUFREG, %%NROUNDS, \ + %%CNTR_TYPE, %%RBITS + jmp %%_small_initial_blocks_encrypted + +%%_small_initial_num_blocks_is_6: + INITIAL_BLOCKS_PARTIAL %%KEY, %%CYPH_PLAIN_OUT, \ + %%PLAIN_CYPH_IN, %%LENGTH, 6, \ + %%CTR, \ + %%ZTMP1, %%ZTMP2, %%ZTMP3, %%ZTMP4, %%ZTMP5, \ + %%ZTMP6, %%ZTMP7, %%ZTMP8, \ + %%IA0, %%IA1, %%MASKREG, %%SHUFREG, %%NROUNDS, \ + %%CNTR_TYPE, %%RBITS + jmp %%_small_initial_blocks_encrypted + +%%_small_initial_num_blocks_is_5: + INITIAL_BLOCKS_PARTIAL %%KEY, %%CYPH_PLAIN_OUT, \ + %%PLAIN_CYPH_IN, %%LENGTH, 5, \ + %%CTR, \ + %%ZTMP1, %%ZTMP2, %%ZTMP3, %%ZTMP4, %%ZTMP5, \ + %%ZTMP6, %%ZTMP7, %%ZTMP8, \ + %%IA0, %%IA1, %%MASKREG, %%SHUFREG, %%NROUNDS, \ + %%CNTR_TYPE, %%RBITS + jmp %%_small_initial_blocks_encrypted + +%%_small_initial_num_blocks_is_4: + INITIAL_BLOCKS_PARTIAL %%KEY, %%CYPH_PLAIN_OUT, \ + %%PLAIN_CYPH_IN, %%LENGTH, 4, \ + %%CTR, \ + %%ZTMP1, %%ZTMP2, %%ZTMP3, %%ZTMP4, %%ZTMP5, \ + %%ZTMP6, %%ZTMP7, %%ZTMP8, \ + %%IA0, %%IA1, %%MASKREG, %%SHUFREG, %%NROUNDS, \ + %%CNTR_TYPE, %%RBITS + jmp %%_small_initial_blocks_encrypted + +%%_small_initial_num_blocks_is_3: + INITIAL_BLOCKS_PARTIAL %%KEY, %%CYPH_PLAIN_OUT, \ + %%PLAIN_CYPH_IN, %%LENGTH, 3, \ + %%CTR, \ + %%ZTMP1, %%ZTMP2, %%ZTMP3, %%ZTMP4, %%ZTMP5, \ + %%ZTMP6, %%ZTMP7, %%ZTMP8, \ + %%IA0, %%IA1, %%MASKREG, %%SHUFREG, %%NROUNDS, \ + %%CNTR_TYPE, %%RBITS + jmp %%_small_initial_blocks_encrypted + +%%_small_initial_num_blocks_is_2: + INITIAL_BLOCKS_PARTIAL %%KEY, %%CYPH_PLAIN_OUT, \ + %%PLAIN_CYPH_IN, %%LENGTH, 2, \ + %%CTR, \ + %%ZTMP1, %%ZTMP2, %%ZTMP3, %%ZTMP4, %%ZTMP5, \ + %%ZTMP6, %%ZTMP7, %%ZTMP8, \ + %%IA0, %%IA1, %%MASKREG, %%SHUFREG, %%NROUNDS, \ + %%CNTR_TYPE, %%RBITS + jmp %%_small_initial_blocks_encrypted + +%%_small_initial_num_blocks_is_1: + INITIAL_BLOCKS_PARTIAL %%KEY, %%CYPH_PLAIN_OUT, \ + %%PLAIN_CYPH_IN, %%LENGTH, 1, \ + %%CTR, \ + %%ZTMP1, %%ZTMP2, %%ZTMP3, %%ZTMP4, %%ZTMP5, \ + %%ZTMP6, %%ZTMP7, %%ZTMP8, \ + %%IA0, %%IA1, %%MASKREG, %%SHUFREG, %%NROUNDS, \ + %%CNTR_TYPE, %%RBITS +%%_small_initial_blocks_encrypted: + +%endmacro ; CNTR_ENC_DEC_SMALL + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; CNTR_ENC_DEC Encodes/Decodes given data. +; Requires the input data be at least 1 byte long because of READ_SMALL_INPUT_DATA. +; Input: job structure and number of AES rounds +; Output: job structure +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +%macro CNTR_ENC_DEC 3 +%define %%JOB %1 ; [in/out] job +%define %%NROUNDS %2 ; [in] number of rounds; numerical value +%define %%CNTR_TYPE %3 ; [in] Type of CNTR operation to do (CNTR/CNTR_BIT) + +%define %%KEY rax +%define %%CYPH_PLAIN_OUT rdx +%define %%PLAIN_CYPH_IN r8 +%define %%LENGTH r9 +%define %%DATA_OFFSET r13 +%define %%RBITS r14 + +%define %%IA0 r10 +%define %%IA1 r11 +%define %%IA2 r12 + +%define %%CTR_BLOCKx xmm0 +%define %%CTR_BLOCK_1_4 zmm1 +%define %%CTR_BLOCK_5_8 zmm2 +%define %%CTR_BLOCK_9_12 zmm3 +%define %%CTR_BLOCK_13_16 zmm4 + +%define %%ZTMP0 zmm5 +%define %%ZTMP1 zmm6 +%define %%ZTMP2 zmm7 +%define %%ZTMP3 zmm8 +%define %%ZTMP4 zmm9 +%define %%ZTMP5 zmm10 +%define %%ZTMP6 zmm11 +%define %%ZTMP7 zmm12 +%define %%SHUFREG zmm13 +%define %%ADD8REG zmm14 + +%define %%MASKREG k1 + +;;; Macro flow: +;;; - calculate the number of 16byte blocks in the message +;;; - process (number of 16byte blocks) mod 16 '%%_initial_num_blocks_is_# .. %%_initial_blocks_encrypted' +;;; - process 16x16 byte blocks at a time until all are done in %%_encrypt_by_16_new + + mov %%LENGTH, [%%JOB + _msg_len_to_cipher] + ;; calculate len + ;; convert bits to bytes (message length in bits for CNTR_BIT) +%ifidn %%CNTR_TYPE, CNTR_BIT + mov %%RBITS, %%LENGTH + add %%LENGTH, 7 + shr %%LENGTH, 3 ; LENGTH will hold number of bytes (including partial byte) + and %%RBITS, 7 ; Get remainder bits in last byte (0-7) +%endif + +%ifidn __OUTPUT_FORMAT__, win64 + cmp %%LENGTH, 0 +%else + or %%LENGTH, %%LENGTH +%endif + je %%_enc_dec_done + + xor %%DATA_OFFSET, %%DATA_OFFSET + + mov %%PLAIN_CYPH_IN, [%%JOB + _src] + add %%PLAIN_CYPH_IN, [%%JOB + _cipher_start_src_offset_in_bytes] + mov %%CYPH_PLAIN_OUT, [%%JOB + _dst] + mov %%KEY, [%%JOB + _aes_enc_key_expanded] + + ;; Prepare round keys (only first 10, due to lack of registers) +%assign i 0 +%rep (%%NROUNDS + 2) + vbroadcastf64x2 ZKEY %+ i, [%%KEY + 16*i] +%assign i (i + 1) +%endrep + + mov %%IA1, [%%JOB + _iv] +%ifidn %%CNTR_TYPE, CNTR + ;; Prepare initial mask to read 12 IV bytes + mov %%IA0, 0x0000_0000_0000_0fff + vmovdqa %%CTR_BLOCKx, [rel initial_12_IV_counter] + mov %%IA2, [%%JOB + _iv_len_in_bytes] + test %%IA2, 16 + ;; Set mask to read 16 IV bytes if iv_len = 16 + cmovnz %%IA0, [rel mask_16_bytes] + + kmovq %%MASKREG, %%IA0 + vmovdqu8 %%CTR_BLOCKx{%%MASKREG}, [%%IA1] +%else ;; CNTR_BIT + ;; Read the full 16 bytes of IV + vmovdqu8 %%CTR_BLOCKx, [%%IA1] +%endif ;; CNTR/CNTR_BIT + + vmovdqa64 %%SHUFREG, [rel SHUF_MASK] + ;; store IV as counter in LE format + vpshufb %%CTR_BLOCKx, XWORD(%%SHUFREG) + + ;; Determine how many blocks to process in INITIAL + mov %%IA1, %%LENGTH + shr %%IA1, 4 + and %%IA1, 0xf + + ;; Process one additional block in INITIAL if there is a partial block + mov %%IA0, %%LENGTH + and %%IA0, 0xf + add %%IA0, 0xf + shr %%IA0, 4 + add %%IA1, %%IA0 + ;; %%IA1 can be in the range from 0 to 16 + + ;; Less than 256B will be handled by the small message code, which + ;; can process up to 16 x blocks (16 bytes each) + cmp %%LENGTH, 256 + jge %%_large_message_path + + CNTR_ENC_DEC_SMALL \ + %%KEY, %%CYPH_PLAIN_OUT, %%PLAIN_CYPH_IN, \ + %%LENGTH, \ + %%IA1, %%CTR_BLOCKx, \ + %%ZTMP0, %%ZTMP1, %%ZTMP2, %%ZTMP3, %%ZTMP4, \ + %%ZTMP5, %%ZTMP6, %%ZTMP7, \ + %%IA0, %%IA2, %%MASKREG, %%SHUFREG, %%NROUNDS, \ + %%CNTR_TYPE, %%RBITS + + jmp %%_enc_dec_done + +%%_large_message_path: + ;; Still, don't allow 16 INITIAL blocks since this will + ;; can be handled by the x16 partial loop. + and %%IA1, 0xf + je %%_initial_num_blocks_is_0 + cmp %%IA1, 15 + je %%_initial_num_blocks_is_15 + cmp %%IA1, 14 + je %%_initial_num_blocks_is_14 + cmp %%IA1, 13 + je %%_initial_num_blocks_is_13 + cmp %%IA1, 12 + je %%_initial_num_blocks_is_12 + cmp %%IA1, 11 + je %%_initial_num_blocks_is_11 + cmp %%IA1, 10 + je %%_initial_num_blocks_is_10 + cmp %%IA1, 9 + je %%_initial_num_blocks_is_9 + cmp %%IA1, 8 + je %%_initial_num_blocks_is_8 + cmp %%IA1, 7 + je %%_initial_num_blocks_is_7 + cmp %%IA1, 6 + je %%_initial_num_blocks_is_6 + cmp %%IA1, 5 + je %%_initial_num_blocks_is_5 + cmp %%IA1, 4 + je %%_initial_num_blocks_is_4 + cmp %%IA1, 3 + je %%_initial_num_blocks_is_3 + cmp %%IA1, 2 + je %%_initial_num_blocks_is_2 + jmp %%_initial_num_blocks_is_1 + + and %%IA1, 0xf + je %%_initial_num_blocks_is_0 + + cmp %%IA1, 8 + je %%_initial_num_blocks_is_8 + jl %%_initial_blocks_is_1_7 + + ; Initial blocks 9-15 + cmp %%IA1, 12 + je %%_initial_num_blocks_is_12 + jl %%_initial_blocks_is_9_11 + + ; Initial blocks 13-15 + cmp %%IA1, 15 + je %%_initial_num_blocks_is_15 + cmp %%IA1, 14 + je %%_initial_num_blocks_is_14 + cmp %%IA1, 13 + je %%_initial_num_blocks_is_13 + +%%_initial_blocks_is_9_11: + cmp %%IA1, 11 + je %%_initial_num_blocks_is_11 + cmp %%IA1, 10 + je %%_initial_num_blocks_is_10 + cmp %%IA1, 9 + je %%_initial_num_blocks_is_9 + +%%_initial_blocks_is_1_7: + cmp %%IA1, 4 + je %%_initial_num_blocks_is_4 + jl %%_initial_blocks_is_1_3 + + ; Initial blocks 5-7 + cmp %%IA1, 7 + je %%_initial_num_blocks_is_7 + cmp %%IA1, 6 + je %%_initial_num_blocks_is_6 + cmp %%IA1, 5 + je %%_initial_num_blocks_is_5 + +%%_initial_blocks_is_1_3: + cmp %%IA1, 3 + je %%_initial_num_blocks_is_3 + cmp %%IA1, 2 + je %%_initial_num_blocks_is_2 + + jmp %%_initial_num_blocks_is_1 + +%%_initial_num_blocks_is_15: + INITIAL_BLOCKS %%KEY, %%CYPH_PLAIN_OUT, %%PLAIN_CYPH_IN, \ + %%LENGTH, %%DATA_OFFSET, 15, %%CTR_BLOCKx, \ + %%CTR_BLOCK_1_4, %%CTR_BLOCK_5_8, %%CTR_BLOCK_9_12, \ + %%CTR_BLOCK_13_16, %%ZTMP0, %%ZTMP1, %%ZTMP2, %%ZTMP3, \ + %%ZTMP4, %%ZTMP5, %%ZTMP6, %%ZTMP7, %%IA0, %%IA1, %%MASKREG, \ + %%SHUFREG, %%NROUNDS, %%CNTR_TYPE, %%RBITS + jmp %%_initial_blocks_encrypted +%%_initial_num_blocks_is_14: + INITIAL_BLOCKS %%KEY, %%CYPH_PLAIN_OUT, %%PLAIN_CYPH_IN, \ + %%LENGTH, %%DATA_OFFSET, 14, %%CTR_BLOCKx, \ + %%CTR_BLOCK_1_4, %%CTR_BLOCK_5_8, %%CTR_BLOCK_9_12, \ + %%CTR_BLOCK_13_16, %%ZTMP0, %%ZTMP1, %%ZTMP2, %%ZTMP3, \ + %%ZTMP4, %%ZTMP5, %%ZTMP6, %%ZTMP7, %%IA0, %%IA1, %%MASKREG, \ + %%SHUFREG, %%NROUNDS, %%CNTR_TYPE, %%RBITS + jmp %%_initial_blocks_encrypted +%%_initial_num_blocks_is_13: + INITIAL_BLOCKS %%KEY, %%CYPH_PLAIN_OUT, %%PLAIN_CYPH_IN, \ + %%LENGTH, %%DATA_OFFSET, 13, %%CTR_BLOCKx, \ + %%CTR_BLOCK_1_4, %%CTR_BLOCK_5_8, %%CTR_BLOCK_9_12, \ + %%CTR_BLOCK_13_16, %%ZTMP0, %%ZTMP1, %%ZTMP2, %%ZTMP3, \ + %%ZTMP4, %%ZTMP5, %%ZTMP6, %%ZTMP7, %%IA0, %%IA1, %%MASKREG, \ + %%SHUFREG, %%NROUNDS, %%CNTR_TYPE, %%RBITS + jmp %%_initial_blocks_encrypted +%%_initial_num_blocks_is_12: + INITIAL_BLOCKS %%KEY, %%CYPH_PLAIN_OUT, %%PLAIN_CYPH_IN, \ + %%LENGTH, %%DATA_OFFSET, 12, %%CTR_BLOCKx, \ + %%CTR_BLOCK_1_4, %%CTR_BLOCK_5_8, %%CTR_BLOCK_9_12, \ + %%CTR_BLOCK_13_16, %%ZTMP0, %%ZTMP1, %%ZTMP2, %%ZTMP3, \ + %%ZTMP4, %%ZTMP5, %%ZTMP6, %%ZTMP7, %%IA0, %%IA1, %%MASKREG, \ + %%SHUFREG, %%NROUNDS, %%CNTR_TYPE, %%RBITS + jmp %%_initial_blocks_encrypted +%%_initial_num_blocks_is_11: + INITIAL_BLOCKS %%KEY, %%CYPH_PLAIN_OUT, %%PLAIN_CYPH_IN, \ + %%LENGTH, %%DATA_OFFSET, 11, %%CTR_BLOCKx, \ + %%CTR_BLOCK_1_4, %%CTR_BLOCK_5_8, %%CTR_BLOCK_9_12, \ + %%CTR_BLOCK_13_16, %%ZTMP0, %%ZTMP1, %%ZTMP2, %%ZTMP3, \ + %%ZTMP4, %%ZTMP5, %%ZTMP6, %%ZTMP7, %%IA0, %%IA1, %%MASKREG, \ + %%SHUFREG, %%NROUNDS, %%CNTR_TYPE, %%RBITS + jmp %%_initial_blocks_encrypted +%%_initial_num_blocks_is_10: + INITIAL_BLOCKS %%KEY, %%CYPH_PLAIN_OUT, %%PLAIN_CYPH_IN, \ + %%LENGTH, %%DATA_OFFSET, 10, %%CTR_BLOCKx, \ + %%CTR_BLOCK_1_4, %%CTR_BLOCK_5_8, %%CTR_BLOCK_9_12, \ + %%CTR_BLOCK_13_16, %%ZTMP0, %%ZTMP1, %%ZTMP2, %%ZTMP3, \ + %%ZTMP4, %%ZTMP5, %%ZTMP6, %%ZTMP7, %%IA0, %%IA1, %%MASKREG, \ + %%SHUFREG, %%NROUNDS, %%CNTR_TYPE, %%RBITS + jmp %%_initial_blocks_encrypted +%%_initial_num_blocks_is_9: + INITIAL_BLOCKS %%KEY, %%CYPH_PLAIN_OUT, %%PLAIN_CYPH_IN, \ + %%LENGTH, %%DATA_OFFSET, 9, %%CTR_BLOCKx, \ + %%CTR_BLOCK_1_4, %%CTR_BLOCK_5_8, %%CTR_BLOCK_9_12, \ + %%CTR_BLOCK_13_16, %%ZTMP0, %%ZTMP1, %%ZTMP2, %%ZTMP3, \ + %%ZTMP4, %%ZTMP5, %%ZTMP6, %%ZTMP7, %%IA0, %%IA1, %%MASKREG, \ + %%SHUFREG, %%NROUNDS, %%CNTR_TYPE, %%RBITS + jmp %%_initial_blocks_encrypted +%%_initial_num_blocks_is_8: + INITIAL_BLOCKS %%KEY, %%CYPH_PLAIN_OUT, %%PLAIN_CYPH_IN, \ + %%LENGTH, %%DATA_OFFSET, 8, %%CTR_BLOCKx, \ + %%CTR_BLOCK_1_4, %%CTR_BLOCK_5_8, %%CTR_BLOCK_9_12, \ + %%CTR_BLOCK_13_16, %%ZTMP0, %%ZTMP1, %%ZTMP2, %%ZTMP3, \ + %%ZTMP4, %%ZTMP5, %%ZTMP6, %%ZTMP7, %%IA0, %%IA1, %%MASKREG, \ + %%SHUFREG, %%NROUNDS, %%CNTR_TYPE, %%RBITS + jmp %%_initial_blocks_encrypted +%%_initial_num_blocks_is_7: + INITIAL_BLOCKS %%KEY, %%CYPH_PLAIN_OUT, %%PLAIN_CYPH_IN, \ + %%LENGTH, %%DATA_OFFSET, 7, %%CTR_BLOCKx, \ + %%CTR_BLOCK_1_4, %%CTR_BLOCK_5_8, %%CTR_BLOCK_9_12, \ + %%CTR_BLOCK_13_16, %%ZTMP0, %%ZTMP1, %%ZTMP2, %%ZTMP3, \ + %%ZTMP4, %%ZTMP5, %%ZTMP6, %%ZTMP7, %%IA0, %%IA1, %%MASKREG, \ + %%SHUFREG, %%NROUNDS, %%CNTR_TYPE, %%RBITS + jmp %%_initial_blocks_encrypted +%%_initial_num_blocks_is_6: + INITIAL_BLOCKS %%KEY, %%CYPH_PLAIN_OUT, %%PLAIN_CYPH_IN, \ + %%LENGTH, %%DATA_OFFSET, 6, %%CTR_BLOCKx, \ + %%CTR_BLOCK_1_4, %%CTR_BLOCK_5_8, %%CTR_BLOCK_9_12, \ + %%CTR_BLOCK_13_16, %%ZTMP0, %%ZTMP1, %%ZTMP2, %%ZTMP3, \ + %%ZTMP4, %%ZTMP5, %%ZTMP6, %%ZTMP7, %%IA0, %%IA1, %%MASKREG, \ + %%SHUFREG, %%NROUNDS, %%CNTR_TYPE, %%RBITS + jmp %%_initial_blocks_encrypted +%%_initial_num_blocks_is_5: + INITIAL_BLOCKS %%KEY, %%CYPH_PLAIN_OUT, %%PLAIN_CYPH_IN, \ + %%LENGTH, %%DATA_OFFSET, 5, %%CTR_BLOCKx, \ + %%CTR_BLOCK_1_4, %%CTR_BLOCK_5_8, %%CTR_BLOCK_9_12, \ + %%CTR_BLOCK_13_16, %%ZTMP0, %%ZTMP1, %%ZTMP2, %%ZTMP3, \ + %%ZTMP4, %%ZTMP5, %%ZTMP6, %%ZTMP7, %%IA0, %%IA1, %%MASKREG, \ + %%SHUFREG, %%NROUNDS, %%CNTR_TYPE, %%RBITS + jmp %%_initial_blocks_encrypted +%%_initial_num_blocks_is_4: + INITIAL_BLOCKS %%KEY, %%CYPH_PLAIN_OUT, %%PLAIN_CYPH_IN, \ + %%LENGTH, %%DATA_OFFSET, 4, %%CTR_BLOCKx, \ + %%CTR_BLOCK_1_4, %%CTR_BLOCK_5_8, %%CTR_BLOCK_9_12, \ + %%CTR_BLOCK_13_16, %%ZTMP0, %%ZTMP1, %%ZTMP2, %%ZTMP3, \ + %%ZTMP4, %%ZTMP5, %%ZTMP6, %%ZTMP7, %%IA0, %%IA1, %%MASKREG, \ + %%SHUFREG, %%NROUNDS, %%CNTR_TYPE, %%RBITS + jmp %%_initial_blocks_encrypted +%%_initial_num_blocks_is_3: + INITIAL_BLOCKS %%KEY, %%CYPH_PLAIN_OUT, %%PLAIN_CYPH_IN, \ + %%LENGTH, %%DATA_OFFSET, 3, %%CTR_BLOCKx, \ + %%CTR_BLOCK_1_4, %%CTR_BLOCK_5_8, %%CTR_BLOCK_9_12, \ + %%CTR_BLOCK_13_16, %%ZTMP0, %%ZTMP1, %%ZTMP2, %%ZTMP3, \ + %%ZTMP4, %%ZTMP5, %%ZTMP6, %%ZTMP7, %%IA0, %%IA1, %%MASKREG, \ + %%SHUFREG, %%NROUNDS, %%CNTR_TYPE, %%RBITS + jmp %%_initial_blocks_encrypted +%%_initial_num_blocks_is_2: + INITIAL_BLOCKS %%KEY, %%CYPH_PLAIN_OUT, %%PLAIN_CYPH_IN, \ + %%LENGTH, %%DATA_OFFSET, 2, %%CTR_BLOCKx, \ + %%CTR_BLOCK_1_4, %%CTR_BLOCK_5_8, %%CTR_BLOCK_9_12, \ + %%CTR_BLOCK_13_16, %%ZTMP0, %%ZTMP1, %%ZTMP2, %%ZTMP3, \ + %%ZTMP4, %%ZTMP5, %%ZTMP6, %%ZTMP7, %%IA0, %%IA1, %%MASKREG, \ + %%SHUFREG, %%NROUNDS, %%CNTR_TYPE, %%RBITS + jmp %%_initial_blocks_encrypted +%%_initial_num_blocks_is_1: + INITIAL_BLOCKS %%KEY, %%CYPH_PLAIN_OUT, %%PLAIN_CYPH_IN, \ + %%LENGTH, %%DATA_OFFSET, 1, %%CTR_BLOCKx, \ + %%CTR_BLOCK_1_4, %%CTR_BLOCK_5_8, %%CTR_BLOCK_9_12, \ + %%CTR_BLOCK_13_16, %%ZTMP0, %%ZTMP1, %%ZTMP2, %%ZTMP3, \ + %%ZTMP4, %%ZTMP5, %%ZTMP6, %%ZTMP7, %%IA0, %%IA1, %%MASKREG, \ + %%SHUFREG, %%NROUNDS, %%CNTR_TYPE, %%RBITS + jmp %%_initial_blocks_encrypted +%%_initial_num_blocks_is_0: + INITIAL_BLOCKS %%KEY, %%CYPH_PLAIN_OUT, %%PLAIN_CYPH_IN, \ + %%LENGTH, %%DATA_OFFSET, 0, %%CTR_BLOCKx, \ + %%CTR_BLOCK_1_4, %%CTR_BLOCK_5_8, %%CTR_BLOCK_9_12, \ + %%CTR_BLOCK_13_16, %%ZTMP0, %%ZTMP1, %%ZTMP2, %%ZTMP3, \ + %%ZTMP4, %%ZTMP5, %%ZTMP6, %%ZTMP7, %%IA0, %%IA1, %%MASKREG, \ + %%SHUFREG, %%NROUNDS, %%CNTR_TYPE, %%RBITS + +%%_initial_blocks_encrypted: + or %%LENGTH, %%LENGTH + je %%_enc_dec_done + + vmovdqa64 %%ADD8REG, [rel ddq_add_16] + ;; Process 15 full blocks plus a partial block + cmp %%LENGTH, 256 + jl %%_encrypt_by_16_partial + +%%_encrypt_by_16: + ENCRYPT_16_PARALLEL %%KEY, %%CYPH_PLAIN_OUT, %%PLAIN_CYPH_IN, \ + %%DATA_OFFSET, %%CTR_BLOCK_1_4, %%CTR_BLOCK_5_8, \ + %%CTR_BLOCK_9_12, %%CTR_BLOCK_13_16, \ + full, %%IA0, %%IA1, %%LENGTH, \ + %%ZTMP0, %%ZTMP1, %%ZTMP2, %%ZTMP3, %%ZTMP4, \ + %%ZTMP5, %%ZTMP6, %%ZTMP7, \ + %%MASKREG, %%SHUFREG, %%ADD8REG, %%NROUNDS, %%CNTR_TYPE, \ + %%RBITS + add %%DATA_OFFSET, 256 + sub %%LENGTH, 256 + cmp %%LENGTH, 256 + jge %%_encrypt_by_16 + +%%_encrypt_by_16_done: + ;; Test to see if we need a by 16 with partial block. At this point + ;; bytes remaining should be either zero or between 241-255. + or %%LENGTH, %%LENGTH + je %%_enc_dec_done + +%%_encrypt_by_16_partial: + + ENCRYPT_16_PARALLEL %%KEY, %%CYPH_PLAIN_OUT, %%PLAIN_CYPH_IN, \ + %%DATA_OFFSET, %%CTR_BLOCK_1_4, %%CTR_BLOCK_5_8, \ + %%CTR_BLOCK_9_12, %%CTR_BLOCK_13_16, \ + partial, %%IA0, %%IA1, %%LENGTH, \ + %%ZTMP0, %%ZTMP1, %%ZTMP2, %%ZTMP3, %%ZTMP4, \ + %%ZTMP5, %%ZTMP6, %%ZTMP7, \ + %%MASKREG, %%SHUFREG, %%ADD8REG, %%NROUNDS, %%CNTR_TYPE, \ + %%RBITS + +%%_enc_dec_done: + +%endmacro ; CNTR_ENC_DEC + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;void aes_cntr_128_submit_vaes_avx512 (JOB_AES_HMAC *job) +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +MKGLOBAL(aes_cntr_128_submit_vaes_avx512,function,internal) +aes_cntr_128_submit_vaes_avx512: + FUNC_SAVE CNTR + ;; arg1 - [in] job + ;; arg2 - [in] NROUNDS + ;; arg3 - [in] Type of CNTR operation to do (CNTR/CNTR_BIT) + CNTR_ENC_DEC arg1, 9, CNTR + FUNC_RESTORE CNTR + + ret + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;void aes_cntr_192_submit_vaes_avx512 (JOB_AES_HMAC *job) +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +MKGLOBAL(aes_cntr_192_submit_vaes_avx512,function,internal) +aes_cntr_192_submit_vaes_avx512: + FUNC_SAVE CNTR + ;; arg1 - [in] job + ;; arg2 - [in] NROUNDS + ;; arg3 - [in] Type of CNTR operation to do (CNTR/CNTR_BIT) + CNTR_ENC_DEC arg1, 11, CNTR + FUNC_RESTORE CNTR + + ret + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;void aes_cntr_256_submit_vaes_avx512 (JOB_AES_HMAC *job) +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +MKGLOBAL(aes_cntr_256_submit_vaes_avx512,function,internal) +aes_cntr_256_submit_vaes_avx512: + FUNC_SAVE CNTR + ;; arg1 - [in] job + ;; arg2 - [in] NROUNDS + ;; arg3 - [in] Type of CNTR operation to do (CNTR/CNTR_BIT) + CNTR_ENC_DEC arg1, 13, CNTR + FUNC_RESTORE CNTR + + ret + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;void aes_cntr_bit_128_submit_vaes_avx512 (JOB_AES_HMAC *job) +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +MKGLOBAL(aes_cntr_bit_128_submit_vaes_avx512,function,internal) +aes_cntr_bit_128_submit_vaes_avx512: + FUNC_SAVE CNTR_BIT + ;; arg1 - [in] job + ;; arg2 - [in] NROUNDS + ;; arg3 - [in] Type of CNTR operation to do (CNTR/CNTR_BIT) + CNTR_ENC_DEC arg1, 9, CNTR_BIT + FUNC_RESTORE CNTR_BIT + + ret + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;void aes_cntr_bit_192_submit_vaes_avx512 (JOB_AES_HMAC *job) +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +MKGLOBAL(aes_cntr_bit_192_submit_vaes_avx512,function,internal) +aes_cntr_bit_192_submit_vaes_avx512: + FUNC_SAVE CNTR_BIT + ;; arg1 - [in] job + ;; arg2 - [in] NROUNDS + ;; arg3 - [in] Type of CNTR operation to do (CNTR/CNTR_BIT) + CNTR_ENC_DEC arg1, 11, CNTR_BIT + FUNC_RESTORE CNTR_BIT + + ret + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;void aes_cntr_bit_256_submit_vaes_avx512 (JOB_AES_HMAC *job) +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +MKGLOBAL(aes_cntr_bit_256_submit_vaes_avx512,function,internal) +aes_cntr_bit_256_submit_vaes_avx512: + FUNC_SAVE CNTR_BIT + ;; arg1 - [in] job + ;; arg2 - [in] NROUNDS + ;; arg3 - [in] Type of CNTR operation to do (CNTR/CNTR_BIT) + CNTR_ENC_DEC arg1, 13, CNTR_BIT + FUNC_RESTORE CNTR_BIT + + ret + +%ifdef LINUX +section .note.GNU-stack noalloc noexec nowrite progbits +%endif diff --git a/src/spdk/intel-ipsec-mb/avx512/des_x16_avx512.asm b/src/spdk/intel-ipsec-mb/avx512/des_x16_avx512.asm new file mode 100644 index 000000000..656752941 --- /dev/null +++ b/src/spdk/intel-ipsec-mb/avx512/des_x16_avx512.asm @@ -0,0 +1,2382 @@ +;; +;; Copyright (c) 2017-2019, Intel Corporation +;; +;; Redistribution and use in source and binary forms, with or without +;; modification, are permitted provided that the following conditions are met: +;; +;; * Redistributions of source code must retain the above copyright notice, +;; this list of conditions and the following disclaimer. +;; * Redistributions in binary form must reproduce the above copyright +;; notice, this list of conditions and the following disclaimer in the +;; documentation and/or other materials provided with the distribution. +;; * Neither the name of Intel Corporation nor the names of its contributors +;; may be used to endorse or promote products derived from this software +;; without specific prior written permission. +;; +;; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +;; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +;; IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +;; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE +;; FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +;; DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +;; SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +;; CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +;; OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +;; + +;; Authors: +;; Shay Gueron (1, 2), Regev Shemy (2), Tomasz kantecki (2) +;; (1) University of Haifa, Israel +;; (2) Intel Corporation + +;; In System V AMD64 ABI +;; calle saves: RBX, RBP, R12-R15 +;; Windows x64 ABI +;; calle saves: RBX, RBP, RDI, RSI, RSP, R12-R15 + +;; +;; Registers: RAX RBX RCX RDX RBP RSI RDI R8 R9 R10 R11 R12 R13 R14 R15 +;; ----------------------------------------------------------- +;; Windows clobbers: RAX R8 R9 R10 R11 +;; Windows preserves: RBX RCX RDX RBP RSI RDI R12 R13 R14 R15 +;; ----------------------------------------------------------- +;; Linux clobbers: RAX RCX RDX R10 R11 +;; Linux preserves: RBX RBP RSI RDI R8 R9 R12 R13 R14 R15 +;; ----------------------------------------------------------- +;; Clobbers ZMM0-31 and K1 to K7 + +%include "include/os.asm" +%include "include/reg_sizes.asm" +%include "mb_mgr_datastruct.asm" +%include "constants.asm" +;%define DO_DBGPRINT +%include "include/dbgprint.asm" + +%ifdef LINUX +%define arg1 rdi +%define arg2 rsi +%define arg3 rdx +%define arg4 rcx +%else +%define arg1 rcx +%define arg2 rdx +%define arg3 r8 +%define arg4 r9 +%endif + +%define STATE arg1 +%define SIZE arg2 + +%define OFFSET rax + +%define IA0 arg3 +%define IA1 arg4 +%define IA2 r10 + +%define INP0 r11 +%define INP1 r12 +%define INP2 r13 +%define INP3 r14 +%define INP4 r15 + +%define KSOFFSET r11 + +%define ZW0 zmm0 +%define ZW1 zmm1 +%define ZW2 zmm2 +%define ZW3 zmm3 +%define ZW4 zmm4 +%define ZW5 zmm5 +%define ZW6 zmm6 +%define ZW7 zmm7 +%define ZW8 zmm8 +%define ZW9 zmm9 +%define ZW10 zmm10 +%define ZW11 zmm11 +%define ZW12 zmm12 +%define ZW13 zmm13 +%define ZW14 zmm14 +%define ZW15 zmm15 + +%define ZIV0 zmm16 +%define ZIV1 zmm17 + +%define ZTMP0 zmm18 +%define ZTMP1 zmm19 +%define ZTMP2 zmm20 +%define ZTMP3 zmm21 +%define ZTMP4 zmm22 +%define ZTMP5 zmm23 +%define ZTMP6 zmm24 +%define ZTMP7 zmm25 +%define ZTMP8 zmm26 +%define ZTMP9 zmm27 +%define ZTMP10 zmm28 +%define ZTMP11 zmm29 +%define ZTMP12 zmm30 +%define ZTMP13 zmm31 + +struc STACKFRAME +_key_sched: resq 16*16 ; 16 lanes x 16 qwords; 16 x 128 bytes = 2048 +_key_sched2: resq 16*16 ; 16 lanes x 16 qwords; 16 x 128 bytes = 2048 +_key_sched3: resq 16*16 ; 16 lanes x 16 qwords; 16 x 128 bytes = 2048 +_tmp_iv: resq 16 ; 2 x 64 bytes +_tmp_in: resq 16 ; 2 x 64 bytes +_tmp_out: resq 16 ; 2 x 64 bytes +_tmp_mask: resd 16 ; 1 x 64 bytes +_gpr_save: resq 4 ; r12 to r15 +_rsp_save: resq 1 +_mask_save: resq 1 +_size_save: resq 1 +endstruc + +;;; =========================================================================== +;;; =========================================================================== +;;; MACROS +;;; =========================================================================== +;;; =========================================================================== + +;;; =========================================================================== +;;; CLEAR TRANSPOSED KEY SCHEDULE (if SAFE_DATA is selected) +;;; =========================================================================== +%macro CLEAR_KEY_SCHEDULE 2 +%define %%ALG %1 ; [in] DES or 3DES +%define %%ZT %2 ; [clobbered] temporary ZMM register + +%ifdef SAFE_DATA + vpxorq %%ZT, %%ZT +%assign rep_num (2048 / 64) +%ifidn %%ALG, 3DES +%assign rep_num (rep_num * 3) +%endif + +%assign offset 0 +%rep rep_num + vmovdqa64 [rsp + _key_sched + offset], %%ZT +%assign offset (offset + 64) +%endrep + +%endif ; SAFE_DATA + +%endmacro + +;;; =========================================================================== +;;; PERMUTE +;;; =========================================================================== +;;; A [in/out] - zmm register +;;; B [in/out] - zmm register +;;; NSHIFT [in] - constant to shift words by +;;; MASK [in] - zmm or m512 with mask +;;; T0 [clobbered] - temporary zmm register +%macro PERMUTE 5 +%define %%A %1 +%define %%B %2 +%define %%NSHIFT %3 +%define %%MASK %4 +%define %%T0 %5 + + vpsrld %%T0, %%A, %%NSHIFT + vpxord %%T0, %%T0, %%B + vpandd %%T0, %%T0, %%MASK + vpxord %%B, %%B, %%T0 + vpslld %%T0, %%T0, %%NSHIFT + vpxord %%A, %%A, %%T0 +%endmacro + +;;; =========================================================================== +;;; INITIAL PERMUTATION +;;; =========================================================================== +;;; L [in/out] - zmm register +;;; R [in/out] - zmm register +;;; T0 [clobbered] - temporary zmm register +%macro IP_Z 3 +%define %%L %1 +%define %%R %2 +%define %%T0 %3 + PERMUTE %%R, %%L, 4, [rel init_perm_consts + 0*64], %%T0 + PERMUTE %%L, %%R, 16, [rel init_perm_consts + 1*64], %%T0 + PERMUTE %%R, %%L, 2, [rel init_perm_consts + 2*64], %%T0 + PERMUTE %%L, %%R, 8, [rel init_perm_consts + 3*64], %%T0 + PERMUTE %%R, %%L, 1, [rel init_perm_consts + 4*64], %%T0 +%endmacro + +;;; =========================================================================== +;;; FINAL PERMUTATION +;;; =========================================================================== +;;; L [in/out] - zmm register +;;; R [in/out] - zmm register +;;; T0 [clobbered] - temporary zmm register +%macro FP_Z 3 +%define %%L %1 +%define %%R %2 +%define %%T0 %3 + PERMUTE %%L, %%R, 1, [rel init_perm_consts + 4*64], %%T0 + PERMUTE %%R, %%L, 8, [rel init_perm_consts + 3*64], %%T0 + PERMUTE %%L, %%R, 2, [rel init_perm_consts + 2*64], %%T0 + PERMUTE %%R, %%L, 16, [rel init_perm_consts + 1*64], %%T0 + PERMUTE %%L, %%R, 4, [rel init_perm_consts + 0*64], %%T0 +%endmacro + +;;; =========================================================================== +;;; P PHASE +;;; =========================================================================== +;;; W0 [in/out] - zmm register +;;; in: vector of 16 x 32bits from S phase +;;; out: permuted in vector +;;; T0-T3 [clobbered] - temporary zmm register +%macro P_PHASE 5 +%define %%W0 %1 +%define %%T0 %2 +%define %%T1 %3 +%define %%T2 %4 +%define %%T3 %5 + + vprord %%T0, %%W0, 3 + vpandd %%T0, %%T0, [rel mask_values + 0*64] + vprord %%T1, %%W0, 5 + vpandd %%T1, %%T1, [rel mask_values + 1*64] + vpord %%T0, %%T0, %%T1 + + vprord %%T1, %%W0, 24 + vpandd %%T1, %%T1, [rel mask_values + 2*64] + vprord %%T2, %%W0, 26 + vpandd %%T2, %%T2, [rel mask_values + 3*64] + vpord %%T1, %%T1, %%T2 + vpord %%T0, %%T0, %%T1 + + vprord %%T1, %%W0, 15 + vpandd %%T1, %%T1, [rel mask_values + 4*64] + vprord %%T2, %%W0, 17 + vpandd %%T2, %%T2, [rel mask_values + 5*64] + vpord %%T1, %%T1, %%T2 + + vprord %%T2, %%W0, 6 + vpandd %%T2, %%T2, [rel mask_values + 6*64] + vprord %%T3, %%W0, 21 + vpandd %%T3, %%T3, [rel mask_values + 7*64] + vpord %%T2, %%T2, %%T3 + vpord %%T1, %%T1, %%T2 + vpord %%T0, %%T0, %%T1 + + vprord %%T1, %%W0, 12 + vpandd %%T1, %%T1, [rel mask_values + 8*64] + vprord %%T2, %%W0, 14 + vpandd %%T2, %%T2, [rel mask_values + 9*64] + vpord %%T1, %%T1, %%T2 + + vprord %%T2, %%W0, 4 + vpandd %%T2, %%T2, [rel mask_values + 10*64] + vprord %%T3, %%W0, 11 + vpandd %%T3, %%T3, [rel mask_values + 11*64] + vpord %%T2, %%T2, %%T3 + vpord %%T1, %%T1, %%T2 + vpord %%T0, %%T0, %%T1 + + vprord %%T1, %%W0, 16 + vpandd %%T1, %%T1, [rel mask_values + 12*64] + vprord %%T2, %%W0, 22 + vpandd %%T2, %%T2, [rel mask_values + 13*64] + vpord %%T1, %%T1, %%T2 + + vprord %%T2, %%W0, 19 + vpandd %%T2, %%T2, [rel mask_values + 14*64] + vprord %%T3, %%W0, 10 + vpandd %%T3, %%T3, [rel mask_values + 15*64] + vpord %%T2, %%T2, %%T3 + vpord %%T1, %%T1, %%T2 + vpord %%T0, %%T0, %%T1 + + vprord %%T1, %%W0, 9 + vpandd %%T1, %%T1, [rel mask_values + 16*64] + vprord %%T2, %%W0, 13 + vpandd %%T2, %%T2, [rel mask_values + 17*64] + vpord %%T1, %%T1, %%T2 + + vprord %%T2, %%W0, 25 + vpandd %%T2, %%T2, [rel mask_values + 18*64] + vpord %%T1, %%T1, %%T2 + vpord %%W0, %%T0, %%T1 +%endmacro + +;;; =========================================================================== +;;; E PHASE +;;; =========================================================================== +;;; +;;; Expands 16x32-bit words into 16x48-bit words +;;; plus XOR's result with the key schedule. +;;; The output is adjusted to be friendly as S phase input. +;;; +;;; in [in] - zmm register +;;; out0a [out] - zmm register +;;; out0b [out] - zmm register +;;; out1a [out] - zmm register +;;; out1b [out] - zmm register +;;; k0 [in] - key schedule; zmm or m512 +;;; k1 [in] - key schedule; zmm or m512 +;;; t0-t1 [clobbered] - temporary zmm register +%macro E_PHASE 9 +%define %%IN %1 +%define %%OUT0A %2 +%define %%OUT0B %3 +%define %%OUT1A %4 +%define %%OUT1B %5 +%define %%K0 %6 +%define %%K1 %7 +%define %%T0 %8 +%define %%T1 %9 + + vprord %%T0, %%IN, 31 + vprord %%T1, %%IN, 3 + vpshufb %%T0, %%T0, [rel idx_e] + vpshufb %%T1, %%T1, [rel idx_e] + vpunpcklbw %%OUT0A, %%T0, %%T1 + vpunpckhbw %%OUT1A, %%T0, %%T1 + vpxord %%OUT0A, %%OUT0A, %%K0 + vpxord %%OUT1A, %%OUT1A, %%K1 + vpandd %%OUT0B, %%OUT0A, [rel and_eu] + vpsrlw %%OUT0B, %%OUT0B, 8 + vpandd %%OUT0A, %%OUT0A, [rel and_ed] + vpandd %%OUT1B, %%OUT1A, [rel and_eu] + vpsrlw %%OUT1B, %%OUT1B, 8 + vpandd %%OUT1A, %%OUT1A, [rel and_ed] +%endmacro + +;;; =========================================================================== +;;; S-BOX +;;; =========================================================================== +;;; +;;; NOTE: clobbers k1-k6 OpMask registers +;;; +;;; IN0A [in] - zmm register; output from E-phase +;;; IN0B [in] - zmm register; output from E-phase +;;; IN1A [in] - zmm register; output from E-phase +;;; IN1B [in] - zmm register; output from E-phase +;;; OUT [out] - zmm register; output from E-phase +;;; T0-T5 [clobbered] - temporary zmm register +%macro S_PHASE 11 +%define %%IN0A %1 +%define %%IN0B %2 +%define %%IN1A %3 +%define %%IN1B %4 +%define %%OUT %5 +%define %%T0 %6 +%define %%T1 %7 +%define %%T2 %8 +%define %%T3 %9 +%define %%T4 %10 +%define %%T5 %11 + + vmovdqa64 %%T0, [rel reg_values16bit_7] + vpcmpuw k3, %%IN0A, %%T0, 2 ; 2 -> LE + vpcmpuw k4, %%IN0B, %%T0, 2 ; 2 -> LE + vpcmpuw k5, %%IN1A, %%T0, 2 ; 2 -> LE + vpcmpuw k6, %%IN1B, %%T0, 2 ; 2 -> LE + + mov DWORD(IA0), 0x55555555 + kmovd k1, DWORD(IA0) + mov DWORD(IA0), 0xaaaaaaaa + kmovd k2, DWORD(IA0) + + vpermw %%T0{k1}{z}, %%IN0A, [rel S_box_flipped + 0*64] + vpermw %%T1{k1}{z}, %%IN0A, [rel S_box_flipped + 1*64] + vpermw %%T2{k2}{z}, %%IN0A, [rel S_box_flipped + 4*64] + vpermw %%T3{k2}{z}, %%IN0A, [rel S_box_flipped + 5*64] + vpxord %%T0, %%T0, %%T2 + vpxord %%OUT, %%T1, %%T3 + vmovdqu16 %%OUT{k3}, %%T0 + + vpermw %%T0{k1}{z}, %%IN0B, [rel S_box_flipped + 2*64] + vpermw %%T1{k1}{z}, %%IN0B, [rel S_box_flipped + 3*64] + vpermw %%T2{k2}{z}, %%IN0B, [rel S_box_flipped + 6*64] + vpermw %%T3{k2}{z}, %%IN0B, [rel S_box_flipped + 7*64] + vpxord %%T0, %%T0, %%T2 + vpxord %%T3, %%T1, %%T3 + vmovdqu16 %%T3{k4}, %%T0 + vpsllw %%T3, %%T3, 4 + vpxord %%OUT, %%OUT, %%T3 + + vpermw %%T0{k1}{z}, %%IN1A, [rel S_box_flipped + 8*64] + vpermw %%T1{k1}{z}, %%IN1A, [rel S_box_flipped + 9*64] + vpermw %%T2{k2}{z}, %%IN1A, [rel S_box_flipped + 12*64] + vpermw %%T3{k2}{z}, %%IN1A, [rel S_box_flipped + 13*64] + vpxord %%T0, %%T0, %%T2 + vpxord %%T4, %%T1, %%T3 + vmovdqu16 %%T4{k5}, %%T0 + + vpermw %%T0{k1}{z}, %%IN1B, [rel S_box_flipped + 10*64] + vpermw %%T1{k1}{z}, %%IN1B, [rel S_box_flipped + 11*64] + vpermw %%T2{k2}{z}, %%IN1B, [rel S_box_flipped + 14*64] + vpermw %%T3{k2}{z}, %%IN1B, [rel S_box_flipped + 15*64] + vpxord %%T0, %%T0, %%T2 + vpxord %%T5, %%T1, %%T3 + vmovdqu16 %%T5{k6}, %%T0 + vpsllw %%T5, %%T5, 4 + + vpxord %%T4, %%T4, %%T5 + vpsllw %%T4, %%T4, 8 + vpxord %%OUT, %%OUT, %%T4 + vpshufb %%OUT, %%OUT, [rel shuffle_reg] +%endmacro + +;;; =========================================================================== +;;; DES encryption/decryption round +;;; =========================================================================== +;;; +;;; Clobbers k1-k6 OpMask registers +;;; +;;; ENC_DEC [in] - ENC for encryption, DEC for decryption +;;; R [in/out] - zmm register; plain text in & cipher text out +;;; L [in/out] - zmm register; plain text in & cipher text out +;;; KS [in] - pointer to the key schedule +;;; T0-T11 [clobbered] - temporary zmm register +%macro DES_ENC_DEC 16 +%define %%ENC_DEC %1 +%define %%R %2 +%define %%L %3 +%define %%KS %4 +%define %%T0 %5 +%define %%T1 %6 +%define %%T2 %7 +%define %%T3 %8 +%define %%T4 %9 +%define %%T5 %10 +%define %%T6 %11 +%define %%T7 %12 +%define %%T8 %13 +%define %%T9 %14 +%define %%T10 %15 +%define %%T11 %16 + + IP_Z %%R, %%L, %%T0 + +%ifidn %%ENC_DEC, ENC + ;; ENCRYPTION + xor KSOFFSET, KSOFFSET +%%_des_enc_loop: + E_PHASE %%R, %%T1, %%T2, %%T3, %%T4, [%%KS + KSOFFSET + (0*64)], [%%KS + KSOFFSET + (1*64)], %%T6, %%T7 + S_PHASE %%T1, %%T2, %%T3, %%T4, %%T0, %%T6, %%T7, %%T8, %%T9, %%T10, %%T11 + P_PHASE %%T0, %%T1, %%T2, %%T3, %%T4 + vpxord %%L, %%L, %%T0 + + E_PHASE %%L, %%T1, %%T2, %%T3, %%T4, [%%KS + KSOFFSET + (2*64)], [%%KS + KSOFFSET + (3*64)], %%T6, %%T7 + S_PHASE %%T1, %%T2, %%T3, %%T4, %%T0, %%T6, %%T7, %%T8, %%T9, %%T10, %%T11 + P_PHASE %%T0, %%T1, %%T2, %%T3, %%T4 + vpxord %%R, %%R, %%T0 + + add KSOFFSET, (4*64) + cmp KSOFFSET, (8*(4*64)) + jb %%_des_enc_loop + +%else + ;; DECRYPTION + mov KSOFFSET, (8*(4*64)) +%%_des_dec_loop: + E_PHASE %%R, %%T1, %%T2, %%T3, %%T4, [%%KS + KSOFFSET - (2*64)], [%%KS + KSOFFSET - (1*64)], %%T6, %%T7 + S_PHASE %%T1, %%T2, %%T3, %%T4, %%T0, %%T6, %%T7, %%T8, %%T9, %%T10, %%T11 + P_PHASE %%T0, %%T1, %%T2, %%T3, %%T4 + vpxord %%L, %%L, %%T0 + + E_PHASE %%L, %%T1, %%T2, %%T3, %%T4, [%%KS + KSOFFSET - (4*64)], [%%KS + KSOFFSET - (3*64)], %%T6, %%T7 + S_PHASE %%T1, %%T2, %%T3, %%T4, %%T0, %%T6, %%T7, %%T8, %%T9, %%T10, %%T11 + P_PHASE %%T0, %%T1, %%T2, %%T3, %%T4 + vpxord %%R, %%R, %%T0 + sub KSOFFSET, (4*64) + jnz %%_des_dec_loop +%endif ; DECRYPTION + + FP_Z %%R, %%L, %%T0 +%endmacro + +;;; =========================================================================== +;;; DATA TRANSPOSITION AT DATA INPUT +;;; =========================================================================== +;;; +;;; IN00 - IN15 [in/out]: +;;; in: IN00 - lane 0 data, IN01 - lane 1 data, ... IN15 - lane 15 data +;;; out: R0 - 16 x word0, L0 - 16 x word1, ... L7 - 16 x word15 +;;; T0-T3 [clobbered] - temporary zmm registers +;;; K0-K5 [clobbered] - temporary zmm registers +;;; H0-H3 [clobbered] - temporary zmm registers +%macro TRANSPOSE_IN 30 +%define %%IN00 %1 ; R0 +%define %%IN01 %2 ; L0 +%define %%IN02 %3 ; R1 +%define %%IN03 %4 ; L1 +%define %%IN04 %5 ; R2 +%define %%IN05 %6 ; L2 +%define %%IN06 %7 ; R3 +%define %%IN07 %8 ; L3 +%define %%IN08 %9 ; R4 +%define %%IN09 %10 ; L4 +%define %%IN10 %11 ; R5 +%define %%IN11 %12 ; L5 +%define %%IN12 %13 ; R6 +%define %%IN13 %14 ; L6 +%define %%IN14 %15 ; R7 +%define %%IN15 %16 ; L7 +%define %%T0 %17 +%define %%T1 %18 +%define %%T2 %19 +%define %%T3 %20 +%define %%K0 %21 +%define %%K1 %22 +%define %%K2 %23 +%define %%K3 %24 +%define %%K4 %25 +%define %%K5 %26 +%define %%H0 %27 +%define %%H1 %28 +%define %%H2 %29 +%define %%H3 %30 + + vpunpckldq %%K0, %%IN00, %%IN01 + vpunpckhdq %%K1, %%IN00, %%IN01 + vpunpckldq %%T0, %%IN02, %%IN03 + vpunpckhdq %%T1, %%IN02, %%IN03 + + vpunpckldq %%IN00, %%IN04, %%IN05 + vpunpckhdq %%IN01, %%IN04, %%IN05 + vpunpckldq %%IN02, %%IN06, %%IN07 + vpunpckhdq %%IN03, %%IN06, %%IN07 + + vpunpcklqdq %%K2, %%K0, %%T0 + vpunpckhqdq %%T2, %%K0, %%T0 + vpunpcklqdq %%K3, %%K1, %%T1 + vpunpckhqdq %%T3, %%K1, %%T1 + + vpunpcklqdq %%K0, %%IN00, %%IN02 + vpunpckhqdq %%K1, %%IN00, %%IN02 + vpunpcklqdq %%T0, %%IN01, %%IN03 + vpunpckhqdq %%T1, %%IN01, %%IN03 + + vpunpckldq %%K4, %%IN08, %%IN09 + vpunpckhdq %%K5, %%IN08, %%IN09 + vpunpckldq %%IN04, %%IN10, %%IN11 + vpunpckhdq %%IN05, %%IN10, %%IN11 + vpunpckldq %%IN06, %%IN12, %%IN13 + vpunpckhdq %%IN07, %%IN12, %%IN13 + vpunpckldq %%IN10, %%IN14, %%IN15 + vpunpckhdq %%IN11, %%IN14, %%IN15 + + vpunpcklqdq %%IN12, %%K4, %%IN04 + vpunpckhqdq %%IN13, %%K4, %%IN04 + vpunpcklqdq %%IN14, %%K5, %%IN05 + vpunpckhqdq %%IN15, %%K5, %%IN05 + vpunpcklqdq %%IN00, %%IN06, %%IN10 + vpunpckhqdq %%IN01, %%IN06, %%IN10 + vpunpcklqdq %%IN02, %%IN07, %%IN11 + vpunpckhqdq %%IN03, %%IN07, %%IN11 + + vshufi64x2 %%H0, %%K2, %%K0, 0x44 + vshufi64x2 %%H1, %%K2, %%K0, 0xee + vshufi64x2 %%H2, %%IN12, %%IN00, 0x44 + vshufi64x2 %%H3, %%IN12, %%IN00, 0xee + vshufi64x2 %%IN00, %%H0, %%H2, 0x88 ; R0 + vshufi64x2 %%IN04, %%H0, %%H2, 0xdd ; R2 + vshufi64x2 %%IN08, %%H1, %%H3, 0x88 ; R4 + vshufi64x2 %%IN12, %%H1, %%H3, 0xdd ; R6 + + vshufi64x2 %%H0, %%T2, %%K1, 0x44 + vshufi64x2 %%H1, %%T2, %%K1, 0xee + vshufi64x2 %%H2, %%IN13, %%IN01, 0x44 + vshufi64x2 %%H3, %%IN13, %%IN01, 0xee + vshufi64x2 %%IN01, %%H0, %%H2, 0x88 ; L0 + vshufi64x2 %%IN05, %%H0, %%H2, 0xdd ; L2 + vshufi64x2 %%IN09, %%H1, %%H3, 0x88 ; L4 + vshufi64x2 %%IN13, %%H1, %%H3, 0xdd ; L6 + + vshufi64x2 %%H0, %%K3, %%T0, 0x44 + vshufi64x2 %%H1, %%K3, %%T0, 0xee + vshufi64x2 %%H2, %%IN14, %%IN02, 0x44 + vshufi64x2 %%H3, %%IN14, %%IN02, 0xee + vshufi64x2 %%IN02, %%H0, %%H2, 0x88 ; R1 + vshufi64x2 %%IN06, %%H0, %%H2, 0xdd ; R3 + vshufi64x2 %%IN10, %%H1, %%H3, 0x88 ; R5 + vshufi64x2 %%IN14, %%H1, %%H3, 0xdd ; R7 + + vshufi64x2 %%H0, %%T3, %%T1, 0x44 + vshufi64x2 %%H1, %%T3, %%T1, 0xee + vshufi64x2 %%H2, %%IN15, %%IN03, 0x44 + vshufi64x2 %%H3, %%IN15, %%IN03, 0xee + vshufi64x2 %%IN03, %%H0, %%H2, 0x88 ; L1 + vshufi64x2 %%IN07, %%H0, %%H2, 0xdd ; L3 + vshufi64x2 %%IN11, %%H1, %%H3, 0x88 ; L5 + vshufi64x2 %%IN15, %%H1, %%H3, 0xdd ; L7 +%endmacro + +;;; =========================================================================== +;;; DATA TRANSPOSITION AT DATA OUTPUT +;;; =========================================================================== +;;; +;;; IN00-IN15 aka R0/L0 - R7/L7 [in/out]: +;;; in: R0 - 16 x word0, L0 - 16 x word1, ... L7 - 16 x word15 +;;; out: R0 - lane 0 data, L0 - lane 1 data, ... L7 - lane 15 data +;;; T0-T3 [clobbered] - temporary zmm registers +;;; K0-K5 [clobbered] - temporary zmm registers +;;; H0-H3 [clobbered] - temporary zmm registers +%macro TRANSPOSE_OUT 30 +%define %%IN00 %1 ; R0 +%define %%IN01 %2 ; L0 +%define %%IN02 %3 ; R1 +%define %%IN03 %4 ; L1 +%define %%IN04 %5 ; R2 +%define %%IN05 %6 ; L2 +%define %%IN06 %7 ; R3 +%define %%IN07 %8 ; L3 +%define %%IN08 %9 ; R4 +%define %%IN09 %10 ; L4 +%define %%IN10 %11 ; R5 +%define %%IN11 %12 ; L5 +%define %%IN12 %13 ; R6 +%define %%IN13 %14 ; L6 +%define %%IN14 %15 ; R7 +%define %%IN15 %16 ; L7 +%define %%T0 %17 +%define %%T1 %18 +%define %%T2 %19 +%define %%T3 %20 +%define %%K0 %21 +%define %%K1 %22 +%define %%K2 %23 +%define %%K3 %24 +%define %%K4 %25 +%define %%K5 %26 +%define %%H0 %27 +%define %%H1 %28 +%define %%H2 %29 +%define %%H3 %30 + + vpunpckldq %%K0, %%IN01, %%IN00 + vpunpckhdq %%K1, %%IN01, %%IN00 + vpunpckldq %%T0, %%IN03, %%IN02 + vpunpckhdq %%T1, %%IN03, %%IN02 + + vpunpckldq %%IN00, %%IN05, %%IN04 + vpunpckhdq %%IN01, %%IN05, %%IN04 + vpunpckldq %%IN02, %%IN07, %%IN06 + vpunpckhdq %%IN03, %%IN07, %%IN06 + + vpunpcklqdq %%K2, %%K0, %%T0 + vpunpckhqdq %%T2, %%K0, %%T0 + vpunpcklqdq %%K3, %%K1, %%T1 + vpunpckhqdq %%T3, %%K1, %%T1 + + vpunpcklqdq %%K0, %%IN00, %%IN02 + vpunpckhqdq %%K1, %%IN00, %%IN02 + vpunpcklqdq %%T0, %%IN01, %%IN03 + vpunpckhqdq %%T1, %%IN01, %%IN03 + + vpunpckldq %%K4, %%IN09, %%IN08 + vpunpckhdq %%K5, %%IN09, %%IN08 + vpunpckldq %%IN04, %%IN11, %%IN10 + vpunpckhdq %%IN05, %%IN11, %%IN10 + vpunpckldq %%IN06, %%IN13, %%IN12 + vpunpckhdq %%IN07, %%IN13, %%IN12 + vpunpckldq %%IN10, %%IN15, %%IN14 + vpunpckhdq %%IN11, %%IN15, %%IN14 + + vpunpcklqdq %%IN12, %%K4, %%IN04 + vpunpckhqdq %%IN13, %%K4, %%IN04 + vpunpcklqdq %%IN14, %%K5, %%IN05 + vpunpckhqdq %%IN15, %%K5, %%IN05 + vpunpcklqdq %%IN00, %%IN06, %%IN10 + vpunpckhqdq %%IN01, %%IN06, %%IN10 + vpunpcklqdq %%IN02, %%IN07, %%IN11 + vpunpckhqdq %%IN03, %%IN07, %%IN11 + + vshufi64x2 %%H0, %%K2, %%K0, 0x44 + vshufi64x2 %%H1, %%K2, %%K0, 0xee + vshufi64x2 %%H2, %%IN12, %%IN00, 0x44 + vshufi64x2 %%H3, %%IN12, %%IN00, 0xee + vshufi64x2 %%IN00, %%H0, %%H2, 0x88 ; R0 + vshufi64x2 %%IN04, %%H0, %%H2, 0xdd ; R2 + vshufi64x2 %%IN08, %%H1, %%H3, 0x88 ; R4 + vshufi64x2 %%IN12, %%H1, %%H3, 0xdd ; R6 + + vshufi64x2 %%H0, %%T2, %%K1, 0x44 + vshufi64x2 %%H1, %%T2, %%K1, 0xee + vshufi64x2 %%H2, %%IN13, %%IN01, 0x44 + vshufi64x2 %%H3, %%IN13, %%IN01, 0xee + vshufi64x2 %%IN01, %%H0, %%H2, 0x88 ; L0 + vshufi64x2 %%IN05, %%H0, %%H2, 0xdd ; L2 + vshufi64x2 %%IN09, %%H1, %%H3, 0x88 ; L4 + vshufi64x2 %%IN13, %%H1, %%H3, 0xdd ; L6 + + vshufi64x2 %%H0, %%K3, %%T0, 0x44 + vshufi64x2 %%H1, %%K3, %%T0, 0xee + vshufi64x2 %%H2, %%IN14, %%IN02, 0x44 + vshufi64x2 %%H3, %%IN14, %%IN02, 0xee + vshufi64x2 %%IN02, %%H0, %%H2, 0x88 ; R1 + vshufi64x2 %%IN06, %%H0, %%H2, 0xdd ; R3 + vshufi64x2 %%IN10, %%H1, %%H3, 0x88 ; R5 + vshufi64x2 %%IN14, %%H1, %%H3, 0xdd ; R7 + + vshufi64x2 %%H0, %%T3, %%T1, 0x44 + vshufi64x2 %%H1, %%T3, %%T1, 0xee + vshufi64x2 %%H2, %%IN15, %%IN03, 0x44 + vshufi64x2 %%H3, %%IN15, %%IN03, 0xee + vshufi64x2 %%IN03, %%H0, %%H2, 0x88 ; L1 + vshufi64x2 %%IN07, %%H0, %%H2, 0xdd ; L3 + vshufi64x2 %%IN11, %%H1, %%H3, 0x88 ; L5 + vshufi64x2 %%IN15, %%H1, %%H3, 0xdd ; L7 +%endmacro + +;;; =========================================================================== +;;; DATA TRANSPOSITION OF ONE DES BLOCK AT DATA INPUT +;;; =========================================================================== +;;; +;;; IN00-IN15 / R0/L0-R7/L7 [in/out]: +;;; in: IN00 - lane 0 data, IN01 - lane 1 data, ... IN15 - lane 15 data +;;; out: R0 - 16 x word0, L0 - 16 x word1 +;;; T0,T2 [clobbered] - temporary zmm registers +;;; K0-K4 [clobbered] - temporary zmm registers +;;; H0,H2 [clobbered] - temporary zmm registers +%macro TRANSPOSE_IN_ONE 24 +%define %%IN00 %1 ; R0 +%define %%IN01 %2 ; L0 +%define %%IN02 %3 ; R1 +%define %%IN03 %4 ; L1 +%define %%IN04 %5 ; R2 +%define %%IN05 %6 ; L2 +%define %%IN06 %7 ; R3 +%define %%IN07 %8 ; L3 +%define %%IN08 %9 ; R4 +%define %%IN09 %10 ; L4 +%define %%IN10 %11 ; R5 +%define %%IN11 %12 ; L5 +%define %%IN12 %13 ; R6 +%define %%IN13 %14 ; L6 +%define %%IN14 %15 ; R7 +%define %%IN15 %16 ; L7 +%define %%T0 %17 +%define %%T2 %18 +%define %%K0 %19 +%define %%K1 %20 +%define %%K2 %21 +%define %%K4 %22 +%define %%H0 %23 +%define %%H2 %24 + + vpunpckldq %%K0, %%IN00, %%IN01 + vpunpckhdq %%K1, %%IN00, %%IN01 + vpunpckldq %%T0, %%IN02, %%IN03 + + vpunpckldq %%IN00, %%IN04, %%IN05 + vpunpckhdq %%IN01, %%IN04, %%IN05 + vpunpckldq %%IN02, %%IN06, %%IN07 + + vpunpcklqdq %%K2, %%K0, %%T0 + vpunpckhqdq %%T2, %%K0, %%T0 + + vpunpcklqdq %%K0, %%IN00, %%IN02 + vpunpckhqdq %%K1, %%IN00, %%IN02 + + vpunpckldq %%K4, %%IN08, %%IN09 + vpunpckldq %%IN04, %%IN10, %%IN11 + vpunpckldq %%IN06, %%IN12, %%IN13 + vpunpckldq %%IN10, %%IN14, %%IN15 + + vpunpcklqdq %%IN12, %%K4, %%IN04 + vpunpckhqdq %%IN13, %%K4, %%IN04 + vpunpcklqdq %%IN00, %%IN06, %%IN10 + vpunpckhqdq %%IN01, %%IN06, %%IN10 + + vshufi64x2 %%H0, %%K2, %%K0, 0x44 + vshufi64x2 %%H2, %%IN12, %%IN00, 0x44 + vshufi64x2 %%IN00, %%H0, %%H2, 0x88 ; R0 + + vshufi64x2 %%H0, %%T2, %%K1, 0x44 + vshufi64x2 %%H2, %%IN13, %%IN01, 0x44 + vshufi64x2 %%IN01, %%H0, %%H2, 0x88 ; L0 +%endmacro + +;;; =========================================================================== +;;; DATA TRANSPOSITION OF ONE DES BLOCK AT DATA OUTPUT +;;; =========================================================================== +;;; +;;; IN00-IN15 aka R0/L0 - R7/L7 [in/out]: +;;; in: R0 - 16 x word0, L0 - 16 x word1 +;;; out: R0 - lane 0 data, L0 - lane 1 data, ... L7 - lane 15 data +;;; T0-T3 [clobbered] - temporary zmm registers +;;; K0-K3 [clobbered] - temporary zmm registers +;;; H0,H1 [clobbered] - temporary zmm registers +%macro TRANSPOSE_OUT_ONE 25 +%define %%IN00 %1 ; R0 +%define %%IN01 %2 ; L0 +%define %%IN02 %3 ; R1 +%define %%IN03 %4 ; L1 +%define %%IN04 %5 ; R2 +%define %%IN05 %6 ; L2 +%define %%IN06 %7 ; R3 +%define %%IN07 %8 ; L3 +%define %%IN08 %9 ; R4 +%define %%IN09 %10 ; L4 +%define %%IN10 %11 ; R5 +%define %%IN11 %12 ; L5 +%define %%IN12 %13 ; R6 +%define %%IN13 %14 ; L6 +%define %%IN14 %15 ; R7 +%define %%IN15 %16 ; L7 +%define %%T0 %17 +%define %%T2 %18 +%define %%T3 %19 +%define %%K0 %20 +%define %%K1 %21 +%define %%K2 %22 +%define %%K3 %23 +%define %%H0 %24 +%define %%H1 %25 + + vpxord %%T0, %%T0, %%T0 + + vpunpckldq %%K0, %%IN01, %%IN00 + vpunpckhdq %%K1, %%IN01, %%IN00 + + vpunpcklqdq %%K2, %%K0, %%T0 + vpunpckhqdq %%T2, %%K0, %%T0 + vpunpcklqdq %%K3, %%K1, %%T0 + vpunpckhqdq %%T3, %%K1, %%T0 + + vshufi64x2 %%H0, %%K2, %%T0, 0x44 + vshufi64x2 %%H1, %%K2, %%T0, 0xee + vshufi64x2 %%IN00, %%H0, %%T0, 0x88 ; R0 + vshufi64x2 %%IN04, %%H0, %%T0, 0xdd ; R2 + vshufi64x2 %%IN08, %%H1, %%T0, 0x88 ; R4 + vshufi64x2 %%IN12, %%H1, %%T0, 0xdd ; R6 + + vshufi64x2 %%H0, %%T2, %%T0, 0x44 + vshufi64x2 %%H1, %%T2, %%T0, 0xee + vshufi64x2 %%IN01, %%H0, %%T0, 0x88 ; L0 + vshufi64x2 %%IN05, %%H0, %%T0, 0xdd ; L2 + vshufi64x2 %%IN09, %%H1, %%T0, 0x88 ; L4 + vshufi64x2 %%IN13, %%H1, %%T0, 0xdd ; L6 + + vshufi64x2 %%H0, %%K3, %%T0, 0x44 + vshufi64x2 %%H1, %%K3, %%T0, 0xee + vshufi64x2 %%IN02, %%H0, %%T0, 0x88 ; R1 + vshufi64x2 %%IN06, %%H0, %%T0, 0xdd ; R3 + vshufi64x2 %%IN10, %%H1, %%T0, 0x88 ; R5 + vshufi64x2 %%IN14, %%H1, %%T0, 0xdd ; R7 + + vshufi64x2 %%H0, %%T3, %%T0, 0x44 + vshufi64x2 %%H1, %%T3, %%T0, 0xee + vshufi64x2 %%IN03, %%H0, %%T0, 0x88 ; L1 + vshufi64x2 %%IN07, %%H0, %%T0, 0xdd ; L3 + vshufi64x2 %%IN11, %%H1, %%T0, 0x88 ; L5 + vshufi64x2 %%IN15, %%H1, %%T0, 0xdd ; L7 +%endmacro + +;;; =========================================================================== +;;; DES INITIALIZATION +;;; key schedule transposition and IV set up +;;; =========================================================================== +;;; +;;; STATE_KEYS [in] - KEYS in DES OOO STATE +;;; STATE_IV [ in] - IV in DES OOO STATE +;;; KS [out] - place to store transposed key schedule or NULL +;;; IV0 [out] - r512; initialization vector +;;; IV1 [out] - r512; initialization vector +;;; T0-T27 [clobbered] - temporary r512 +%macro DES_INIT 33 +%define %%STATE_KEYS %1 +%define %%STATE_IV %2 +%define %%KS %3 +%define %%IV0 %4 +%define %%IV1 %5 +%define %%T0 %6 +%define %%T1 %7 +%define %%T2 %8 +%define %%T3 %9 +%define %%T4 %10 +%define %%T5 %11 +%define %%T6 %12 +%define %%T7 %13 +%define %%T8 %14 +%define %%T9 %15 +%define %%T10 %16 +%define %%T11 %17 +%define %%T12 %18 +%define %%T13 %19 +%define %%T14 %20 +%define %%T15 %21 +%define %%T16 %22 +%define %%T17 %23 +%define %%T18 %24 +%define %%T19 %25 +%define %%T20 %26 +%define %%T21 %27 +%define %%T22 %28 +%define %%T23 %29 +%define %%T24 %30 +%define %%T25 %31 +%define %%T26 %32 +%define %%T27 %33 + + ;; set up the key schedule + ;; - load first half of the keys & transpose + ;; - transpose and store + ;; note: we can use IV registers as temprary ones here +%assign IDX 0 +%rep 16 + mov IA0, [%%STATE_KEYS + (IDX*PTR_SZ)] + vmovdqu64 %%T %+ IDX, [IA0] +%assign IDX (IDX + 1) +%endrep + TRANSPOSE_IN %%T0, %%T1, %%T2, %%T3, %%T4, %%T5, %%T6, %%T7, %%T8, %%T9, %%T10, %%T11, %%T12, %%T13, %%T14, %%T15, %%T16, %%T17, %%T18, %%T19, %%T20, %%T21, %%T22, %%T23, %%T24, %%T25, %%T26, %%T27, %%IV0, %%IV1 +%assign IDX 0 +%rep 16 + vmovdqu64 [%%KS + (IDX * 64)], %%T %+ IDX +%assign IDX (IDX + 1) +%endrep + ;; - load second half of the keys & transpose + ;; - transpose and store + ;; note: we can use IV registers as temprary ones here +%assign IDX 0 +%rep 16 + mov IA0, [%%STATE_KEYS + (IDX*PTR_SZ)] + vmovdqu64 %%T %+ IDX, [IA0 + 64] +%assign IDX (IDX + 1) +%endrep + TRANSPOSE_IN %%T0, %%T1, %%T2, %%T3, %%T4, %%T5, %%T6, %%T7, %%T8, %%T9, %%T10, %%T11, %%T12, %%T13, %%T14, %%T15, %%T16, %%T17, %%T18, %%T19, %%T20, %%T21, %%T22, %%T23, %%T24, %%T25, %%T26, %%T27, %%IV0, %%IV1 +%assign IDX 0 +%rep 16 + vmovdqu64 [%%KS + (16 * 64) + (IDX * 64)], %%T %+ IDX +%assign IDX (IDX + 1) +%endrep + + ;; set up IV + ;; - they are already kept transposed so this is enough to load them + vmovdqu64 %%IV0, [%%STATE_IV + (0 * 64)] + vmovdqu64 %%IV1, [%%STATE_IV + (1 * 64)] +%endmacro + +;;; =========================================================================== +;;; 3DES INITIALIZATION +;;; key schedule transposition and IV set up +;;; =========================================================================== +;;; +;;; STATE_KEYS [in] - KEYS in 3DES OOO STATE +;;; STATE_IV [ in] - IV in 3DES OOO STATE +;;; KS1 [out] - place to store transposed key schedule or NULL +;;; KS2 [out] - place to store transposed key schedule or NULL +;;; KS3 [out] - place to store transposed key schedule or NULL +;;; IV0 [out] - r512; initialization vector +;;; IV1 [out] - r512; initialization vector +;;; T0-T27 [clobbered] - temporary r512 +;;; DIR [in] - ENC/DEC (keys arranged in different order for enc/dec) +%macro DES3_INIT 36 +%define %%STATE_KEYS %1 +%define %%STATE_IV %2 +%define %%KS1 %3 +%define %%KS2 %4 +%define %%KS3 %5 +%define %%IV0 %6 +%define %%IV1 %7 +%define %%T0 %8 +%define %%T1 %9 +%define %%T2 %10 +%define %%T3 %11 +%define %%T4 %12 +%define %%T5 %13 +%define %%T6 %14 +%define %%T7 %15 +%define %%T8 %16 +%define %%T9 %17 +%define %%T10 %18 +%define %%T11 %19 +%define %%T12 %20 +%define %%T13 %21 +%define %%T14 %22 +%define %%T15 %23 +%define %%T16 %24 +%define %%T17 %25 +%define %%T18 %26 +%define %%T19 %27 +%define %%T20 %28 +%define %%T21 %29 +%define %%T22 %30 +%define %%T23 %31 +%define %%T24 %32 +%define %%T25 %33 +%define %%T26 %34 +%define %%T27 %35 +%define %%DIR %36 + +%ifidn %%DIR, ENC +%assign KEY_IDX 0 +%else +%assign KEY_IDX 2 +%endif +%assign KS_IDX 1 + +%rep 3 + ;; set up the key schedule + ;; - load first half of the keys & transpose + ;; - transpose and store + ;; note: we can use IV registers as temprary ones here + +%assign IDX 0 +%rep 16 + mov IA0, [%%STATE_KEYS + (IDX*PTR_SZ)] + mov IA0, [IA0 + (KEY_IDX * PTR_SZ)] + vmovdqu64 %%T %+ IDX, [IA0] +%assign IDX (IDX + 1) +%endrep + TRANSPOSE_IN %%T0, %%T1, %%T2, %%T3, %%T4, %%T5, %%T6, %%T7, %%T8, %%T9, %%T10, %%T11, %%T12, %%T13, %%T14, %%T15, %%T16, %%T17, %%T18, %%T19, %%T20, %%T21, %%T22, %%T23, %%T24, %%T25, %%T26, %%T27, %%IV0, %%IV1 +%assign IDX 0 +%rep 16 + vmovdqu64 [%%KS %+ KS_IDX + (IDX * 64)], %%T %+ IDX +%assign IDX (IDX + 1) +%endrep + ;; - load second half of the keys & transpose + ;; - transpose and store + ;; note: we can use IV registers as temprary ones here +%assign IDX 0 +%rep 16 + mov IA0, [%%STATE_KEYS + (IDX*PTR_SZ)] + mov IA0, [IA0 + (KEY_IDX * PTR_SZ)] + vmovdqu64 %%T %+ IDX, [IA0 + 64] +%assign IDX (IDX + 1) +%endrep + TRANSPOSE_IN %%T0, %%T1, %%T2, %%T3, %%T4, %%T5, %%T6, %%T7, %%T8, %%T9, %%T10, %%T11, %%T12, %%T13, %%T14, %%T15, %%T16, %%T17, %%T18, %%T19, %%T20, %%T21, %%T22, %%T23, %%T24, %%T25, %%T26, %%T27, %%IV0, %%IV1 +%assign IDX 0 +%rep 16 + vmovdqu64 [%%KS %+ KS_IDX + (16 * 64) + (IDX * 64)], %%T %+ IDX +%assign IDX (IDX + 1) +%endrep + +%ifidn %%DIR, ENC +%assign KEY_IDX (KEY_IDX + 1) +%else +%assign KEY_IDX (KEY_IDX - 1) +%endif +%assign KS_IDX (KS_IDX + 1) +%endrep ; KEY_IDX / KS_IDX + + ;; set up IV + ;; - they are already kept transposed so this is enough to load them + vmovdqu64 %%IV0, [%%STATE_IV + (0 * 64)] + vmovdqu64 %%IV1, [%%STATE_IV + (1 * 64)] + +%endmacro + +;;; =========================================================================== +;;; DES FINISH +;;; Update in/out pointers and store IV +;;; =========================================================================== +;;; +;;; Needs: STATE & SIZE +;;; IV0 [in] - r512; initialization vector +;;; IV1 [in] - r512; initialization vector +;;; T0-T4 [clobbered] - temporary r512 registers +%macro DES_FINISH 7 +%define %%IV0 %1 +%define %%IV1 %2 +%define %%T0 %3 +%define %%T1 %4 +%define %%T2 %5 +%define %%T3 %6 +%define %%T4 %7 + + vpbroadcastq %%T4, SIZE + vmovdqu64 %%T0, [STATE + _des_args_in + (0 * PTR_SZ)] + vmovdqu64 %%T1, [STATE + _des_args_in + (8 * PTR_SZ)] + vmovdqu64 %%T2, [STATE + _des_args_out + (0 * PTR_SZ)] + vmovdqu64 %%T3, [STATE + _des_args_out + (8 * PTR_SZ)] + vpaddq %%T0, %%T0, %%T4 + vpaddq %%T1, %%T1, %%T4 + vpaddq %%T2, %%T2, %%T4 + vpaddq %%T3, %%T3, %%T4 + vmovdqu64 [STATE + _des_args_in + (0 * PTR_SZ)], %%T0 + vmovdqu64 [STATE + _des_args_in + (8 * PTR_SZ)], %%T1 + vmovdqu64 [STATE + _des_args_out + (0 * PTR_SZ)], %%T2 + vmovdqu64 [STATE + _des_args_out + (8 * PTR_SZ)], %%T3 + + vmovdqu64 [STATE + _des_args_IV + (0 * 64)], %%IV0 + vmovdqu64 [STATE + _des_args_IV + (1 * 64)], %%IV1 +%endmacro + +;;; =========================================================================== +;;; DES CFB ENCRYPT/DECRYPT - ONE BLOCK ONLY +;;; =========================================================================== +;;; +;;; Needs: STATE, IA0-IA2 +;;; ENC_DEC [in] - encyrpt (ENC) or decrypt (DEC) selection +;;; KS [in] - key schedule +;;; T0-T24 [clobbered] - temporary r512 +;;; T_IN [in] - 16 * 8 byte storage +;;; T_OUT [in] - 16 * 8 byte storage +;;; T_MASK [in] - 16 * 4 byte storage +;;; T_IV [in] - 16 * 8 byte storage +;;; +;;; NOTE: clobbers OpMask registers +%macro DES_CFB_ONE 31 +%define %%ENC_DEC %1 +%define %%KS %2 +%define %%T0 %3 +%define %%T1 %4 +%define %%T2 %5 +%define %%T3 %6 +%define %%T4 %7 +%define %%T5 %8 +%define %%T6 %9 +%define %%T7 %10 +%define %%T8 %11 +%define %%T9 %12 +%define %%T10 %13 +%define %%T11 %14 +%define %%T12 %15 +%define %%T13 %16 +%define %%T14 %17 +%define %%T15 %18 +%define %%T16 %19 +%define %%T17 %20 +%define %%T18 %21 +%define %%T19 %22 +%define %%T20 %23 +%define %%T21 %24 +%define %%T22 %25 +%define %%T23 %26 +%define %%T24 %27 +%define %%T_IN %28 +%define %%T_OUT %29 +%define %%T_IV %30 +%define %%T_MASK %31 + + ;; - find mask for non-zero partial lengths + vpxord %%T10, %%T10, %%T10 + vmovdqu64 %%T0, [STATE + _des_args_PLen] + vpcmpd k3, %%T0, %%T10, 4 ; NEQ + kmovw DWORD(IA0), k3 + movzx DWORD(IA0), WORD(IA0) + or DWORD(IA0), DWORD(IA0) + jz %%_des_cfb_one_end ; no non-zero partial lengths + +%ifidn %%ENC_DEC, ENC + ;; For encyrption case we need to make sure that + ;; all full blocks are complete before proceeding + ;; with CFB partial block. + ;; To do that current out position is compared against + ;; calculated last full block position. + vmovdqu64 %%T1, [STATE + _des_args_out + (0*8)] + vmovdqu64 %%T2, [STATE + _des_args_LOut + (0*8)] + vmovdqu64 %%T3, [STATE + _des_args_out + (8*8)] + vmovdqu64 %%T4, [STATE + _des_args_LOut + (8*8)] + vpcmpq k4, %%T1, %%T2, 0 ; EQ + vpcmpq k5, %%T3, %%T4, 0 ; EQ + kmovw DWORD(IA1), k4 + movzx DWORD(IA1), BYTE(IA1) + kmovw DWORD(IA2), k5 + movzx DWORD(IA2), BYTE(IA2) + shl DWORD(IA2), 8 + or DWORD(IA2), DWORD(IA1) + and DWORD(IA0), DWORD(IA2) + jz %%_des_cfb_one_end ; no non-zero lengths left + kmovw k3, DWORD(IA0) +%endif + ;; Calculate ((1 << partial_bytes) - 1) + ;; in order to get the mask for loads and stores + ;; k3 & IA0 - hold valid mask + vmovdqa64 %%T1, [rel vec_ones_32b] + vpsllvd %%T2{k3}{z}, %%T1, %%T0 + vpsubd %%T2{k3}{z}, %%T2, %%T1 + vmovdqu64 [%%T_MASK], %%T2 + + ;; clear selected partial lens not to do them twice + vmovdqu32 [STATE + _des_args_PLen]{k3}, %%T10 + + ;; copy IV, in and out pointers + vmovdqu64 %%T1, [STATE + _des_args_in + (0*PTR_SZ)] + vmovdqu64 %%T2, [STATE + _des_args_in + (8*PTR_SZ)] + vmovdqu64 %%T3, [STATE + _des_args_out + (0*PTR_SZ)] + vmovdqu64 %%T4, [STATE + _des_args_out + (8*PTR_SZ)] + vmovdqu64 %%T5, [STATE + _des_args_IV + (0*64)] + vmovdqu64 %%T6, [STATE + _des_args_IV + (1*64)] + vmovdqu64 [%%T_IN + (0*PTR_SZ)], %%T1 + vmovdqu64 [%%T_IN + (8*PTR_SZ)], %%T2 + vmovdqu64 [%%T_OUT + (0*PTR_SZ)], %%T3 + vmovdqu64 [%%T_OUT + (8*PTR_SZ)], %%T4 + vmovdqu64 [%%T_IV + (0*64)], %%T5 + vmovdqu64 [%%T_IV + (1*64)], %%T6 + + ;; calculate last block case mask + ;; - first block case requires no modifications to in/out/IV + vmovdqu64 %%T1, [STATE + _des_args_BLen] + vpcmpd k2, %%T1, %%T10, 4 ; NEQ + kmovw DWORD(IA1), k2 + and DWORD(IA1), DWORD(IA0) + jz %%_des_cfb_one_no_last_blocks + + ;; set up IV, in and out for the last block case + ;; - Last block needs in and out to be set differently (decryption only) + ;; - IA1 holds the last block mask +%ifidn %%ENC_DEC, DEC + mov DWORD(IA0), DWORD(IA1) + mov DWORD(IA2), DWORD(IA1) + shr DWORD(IA1), 8 + and DWORD(IA2), 0xff + kmovw k4, DWORD(IA2) + kmovw k5, DWORD(IA1) + vmovdqu64 %%T1, [STATE + _des_args_LOut + (0*PTR_SZ)] + vmovdqu64 %%T2, [STATE + _des_args_LOut + (8*PTR_SZ)] + vmovdqu64 %%T3, [STATE + _des_args_LIn + (0*PTR_SZ)] + vmovdqu64 %%T4, [STATE + _des_args_LIn + (8*PTR_SZ)] + vmovdqu64 [%%T_OUT + (0*PTR_SZ)]{k4}, %%T1 + vmovdqu64 [%%T_OUT + (8*PTR_SZ)]{k5}, %%T2 + vmovdqu64 [%%T_IN + (0*PTR_SZ)]{k4}, %%T3 + vmovdqu64 [%%T_IN + (8*PTR_SZ)]{k5}, %%T4 +%endif ; decryption + ;; - IV has to be set differently for CFB as well + ;; - IA0 holds the last block mask +%assign IDX 0 +%rep 16 + test DWORD(IA0), (1 << IDX) + jz %%_des_cfb_one_copy_iv_next %+ IDX +%ifidn %%ENC_DEC, ENC + mov IA2, [STATE + _des_args_LOut + (IDX*PTR_SZ)] +%else + mov IA2, [STATE + _des_args_LIn + (IDX*PTR_SZ)] +%endif + mov IA2, [IA2 - 8] + mov [%%T_IV + (0*4) + (IDX*4)], DWORD(IA2) + shr IA2, 32 + mov [%%T_IV + (16*4) + (IDX*4)], DWORD(IA2) +%%_des_cfb_one_copy_iv_next %+ IDX: +%assign IDX (IDX + 1) +%endrep + +%%_des_cfb_one_no_last_blocks: + ;; Uffff ... finally let's do some DES CFB + ;; - let's use T_IN, T_OUT, T_IV and T_MASK + + ;; - load data with the corresponding masks & transpose + ;; - T0 to T15 will hold the data + xor IA0, IA0 +%assign IDX 0 +%assign K_IDX 1 +%rep 16 + mov IA1, [%%T_IN + (IDX*PTR_SZ)] + mov DWORD(IA0), [%%T_MASK + (IDX*4)] + kmovq k %+ K_IDX, IA0 + vmovdqu8 %%T %+ IDX{k %+ K_IDX}{z}, [IA1] +%assign IDX (IDX + 1) +%assign K_IDX (K_IDX + 1) +%if K_IDX > 7 +%assign K_IDX 1 ; iterate through K1 to K7 +%endif +%endrep + ;; - transpose the data in T0 to T15, T16 to T23 are clobbered + TRANSPOSE_IN_ONE %%T0, %%T1, %%T2, %%T3, %%T4, %%T5, %%T6, %%T7, %%T8, %%T9, %%T10, %%T11, %%T12, %%T13, %%T14, %%T15, %%T16, %%T17, %%T18, %%T19, %%T20, %%T21, %%T22, %%T23 + + ;; - set up IV and %%T16 & %%T17 used as IV0 and IV1 + vmovdqu64 %%T16, [%%T_IV + (0 * 64)] ;IV0 + vmovdqu64 %%T17, [%%T_IV + (1 * 64)] ;IV1 + ;; DES encrypt + ;; - R0 - %%T0 + ;; - L0 - %%T1 + DES_ENC_DEC ENC, %%T16, %%T17, %%KS, %%T2, %%T3, %%T4, %%T5, %%T6, %%T7, %%T8, %%T9, %%T10, %%T11, %%T12, %%T13 + ;; CFB style xor with R0/L0 with IV + ;; - IV0 - %%T16 + ;; - IV1 - %%T17 + vpxord %%T2, %%T17, %%T0 ; R0 ^ IV1 + vpxord %%T0, %%T16, %%T1 ; L0 ^ IV0 + vmovdqa64 %%T1, %%T2 + ;; - new R0 = L0 ^ IV0 (%%T0) + ;; - new L0 = R0 ^ IV1 (%%T1) + + ;; Transpose the data out + ;; - %%T2 to %%T24 clobbered + TRANSPOSE_OUT_ONE %%T0, %%T1, %%T2, %%T3, %%T4, %%T5, %%T6, %%T7, %%T8, %%T9, %%T10, %%T11, %%T12, %%T13, %%T14, %%T15, %%T16, %%T17, %%T18, %%T19, %%T20, %%T21, %%T22, %%T23, %%T24 + + ;; Store the transposed data + ;; - T0 to T15 will hold the data + xor IA0, IA0 +%assign IDX 0 +%assign K_IDX 1 +%rep 16 + mov IA1, [%%T_OUT + (IDX*PTR_SZ)] + mov DWORD(IA0), [%%T_MASK + (IDX*4)] + kmovq k %+ K_IDX, IA0 + vmovdqu8 [IA1]{k %+ K_IDX}, %%T %+ IDX +%assign IDX (IDX + 1) +%assign K_IDX (K_IDX + 1) +%if K_IDX > 7 +%assign K_IDX 1 ; iterate through K1 to K7 +%endif +%endrep + +%ifdef SAFE_DATA + ;; Clear copied IV's + vpxorq %%T5, %%T5 + vmovdqu64 [%%T_IV + (0*64)], %%T5 + vmovdqu64 [%%T_IV + (1*64)], %%T5 +%endif + +%%_des_cfb_one_end: + +%endmacro + +;;; =========================================================================== +;;; Converts length into mask of DES blocks +;;; =========================================================================== +;;; +;;; MASK [out] - mask8 for value; for masked 64b loads and stores (r64) +;;; USES: IA0, IA1 IA2 +;;; ASSUMES: SIZE - OFFSET < 64 +%macro GET_MASK8 1 +%define %%MASK %1 + +%ifidn IA1, rcx +%define myrcx IA1 +%else +%define myrcx rcx + mov IA1, rcx +%endif + mov myrcx, SIZE + sub myrcx, OFFSET + ;; - myrcx - remaining length + ;; - divide by 8 (DES block size) + ;; - create bit mask of the result + mov DWORD(%%MASK), 1 + shr DWORD(myrcx), 3 + shl DWORD(%%MASK), BYTE(myrcx) + sub DWORD(%%MASK), 1 +%ifnidn IA1, rcx + mov rcx, IA1 +%endif +%endmacro + +;;; =========================================================================== +;;; DES CBC ENCRYPT CIPHER ONLY (1 to 8 DES blocks only) +;;; =========================================================================== +;;; +;;; NUM_DES_BLOCKS [in] - 1 to 8 DES blocks only +;;; DES_KS [in] - pointer to transposed key schedule +;;; +;;; NOTE: clobbers OpMask registers +;;; REQUIRES: ZTMP0 - ZTMP13, ZW0-ZW15 (depends on NUM_DES_BLOCKS), ZIV0, ZIV1 +%macro GEN_DES_ENC_CIPHER 2 +%define %%NUM_DES_BLOCKS %1 +%define %%DES_KS %2 + +%assign RN 0 +%assign LN 1 +%assign RNN 2 +%assign LNN 3 +%rep %%NUM_DES_BLOCKS - 1 + DES_ENC_DEC ENC, ZW %+ RN, ZW %+ LN, %%DES_KS, ZTMP0, ZTMP1, ZTMP2, ZTMP3, ZTMP4, ZTMP5, ZTMP6, ZTMP7, ZTMP8, ZTMP9, ZTMP10, ZTMP11 + vpxord ZW %+ RNN, ZW %+ RNN, ZW %+ LN ; R1 = R1 ^ L0 + vpxord ZW %+ LNN, ZW %+ LNN, ZW %+ RN ; L1 = L1 ^ R0 +%assign RN (RN + 2) +%assign LN (LN + 2) +%assign RNN (RNN + 2) +%assign LNN (LNN + 2) +%endrep + DES_ENC_DEC ENC, ZW %+ RN, ZW %+ LN, %%DES_KS, ZTMP0, ZTMP1, ZTMP2, ZTMP3, ZTMP4, ZTMP5, ZTMP6, ZTMP7, ZTMP8, ZTMP9, ZTMP10, ZTMP11 + vmovdqa64 ZIV0, ZW %+ LN ; IV0 = L7 + vmovdqa64 ZIV1, ZW %+ RN ; IV1 = R7 +%endmacro + +;;; =========================================================================== +;;; DES CBC DECRYPT CIPHER ONLY (1 to 8 DES blocks only) +;;; =========================================================================== +;;; +;;; NUM_DES_BLOCKS [in] - 1 to 8 DES blocks only +;;; DES_KS [in] - pointer to transposed key schedule +;;; +;;; NOTE: clobbers OpMask registers +;;; REQUIRES: ZTMP0 - ZTMP13, ZW0-ZW15 (depends on NUM_DES_BLOCKS), ZIV0, ZIV1 +%macro GEN_DES_DEC_CIPHER 2 +%define %%NUM_DES_BLOCKS %1 +%define %%DES_KS %2 + +%assign RN 0 +%assign LN 1 +%rep %%NUM_DES_BLOCKS + vmovdqa64 ZTMP12, ZW %+ RN ; keep R0 as IV for the next round + vmovdqa64 ZTMP13, ZW %+ LN ; keep L0 as IV for the next round + DES_ENC_DEC DEC, ZW %+ RN, ZW %+ LN, %%DES_KS, ZTMP0, ZTMP1, ZTMP2, ZTMP3, ZTMP4, ZTMP5, ZTMP6, ZTMP7, ZTMP8, ZTMP9, ZTMP10, ZTMP11 + vpxord ZW %+ RN, ZW %+ RN, ZIV1 ; R0 = R0 ^ IV1 + vpxord ZW %+ LN, ZW %+ LN, ZIV0 ; L0 = L0 ^ IV0 + vmovdqa64 ZIV0, ZTMP12 + vmovdqa64 ZIV1, ZTMP13 +%assign RN (RN + 2) +%assign LN (LN + 2) +%endrep +%endmacro + +;;; =========================================================================== +;;; 3DES CBC ENCRYPT CIPHER ONLY (1 to 8 DES blocks only) +;;; =========================================================================== +;;; +;;; NUM_DES_BLOCKS [in] - 1 to 8 DES blocks only +;;; DES_KS1 [in] - pointer to transposed key schedule 1 +;;; DES_KS2 [in] - pointer to transposed key schedule 2 +;;; DES_KS3 [in] - pointer to transposed key schedule 3 +;;; +;;; NOTE: clobbers OpMask registers +;;; REQUIRES: ZTMP0 - ZTMP13, ZW0-ZW15 (depends on NUM_DES_BLOCKS), ZIV0, ZIV1 +%macro GEN_3DES_ENC_CIPHER 4 +%define %%NUM_DES_BLOCKS %1 +%define %%DES_KS1 %2 +%define %%DES_KS2 %3 +%define %%DES_KS3 %4 + +%assign RN 0 +%assign LN 1 +%assign RNN 2 +%assign LNN 3 +%rep %%NUM_DES_BLOCKS + ;; ENC + DES_ENC_DEC ENC, ZW %+ RN, ZW %+ LN, %%DES_KS1, ZTMP0, ZTMP1, ZTMP2, ZTMP3, ZTMP4, ZTMP5, ZTMP6, ZTMP7, ZTMP8, ZTMP9, ZTMP10, ZTMP11 + ;; DEC + DES_ENC_DEC DEC, ZW %+ LN, ZW %+ RN, %%DES_KS2, ZTMP0, ZTMP1, ZTMP2, ZTMP3, ZTMP4, ZTMP5, ZTMP6, ZTMP7, ZTMP8, ZTMP9, ZTMP10, ZTMP11 + ;; ENC + DES_ENC_DEC ENC, ZW %+ RN, ZW %+ LN, %%DES_KS3, ZTMP0, ZTMP1, ZTMP2, ZTMP3, ZTMP4, ZTMP5, ZTMP6, ZTMP7, ZTMP8, ZTMP9, ZTMP10, ZTMP11 +%if (RNN < (%%NUM_DES_BLOCKS * 2)) + vpxord ZW %+ RNN, ZW %+ RNN, ZW %+ LN ; R1 = R1 ^ L0 + vpxord ZW %+ LNN, ZW %+ LNN, ZW %+ RN ; L1 = L1 ^ R0 +%else + vmovdqa64 ZIV0, ZW %+ LN ; IV0 = L7 + vmovdqa64 ZIV1, ZW %+ RN ; IV1 = R7 +%endif + +%assign RN (RN + 2) +%assign LN (LN + 2) +%assign RNN (RNN + 2) +%assign LNN (LNN + 2) +%endrep + +%endmacro + +;;; =========================================================================== +;;; 3DES CBC DECRYPT CIPHER ONLY (1 to 8 DES blocks only) +;;; =========================================================================== +;;; +;;; NUM_DES_BLOCKS [in] - 1 to 8 DES blocks only +;;; DES_KS1 [in] - pointer to transposed key schedule 1 +;;; DES_KS2 [in] - pointer to transposed key schedule 2 +;;; DES_KS3 [in] - pointer to transposed key schedule 3 +;;; +;;; NOTE: clobbers OpMask registers +;;; REQUIRES: ZTMP0 - ZTMP13, ZW0-ZW15 (depends on NUM_DES_BLOCKS), ZIV0, ZIV1 +%macro GEN_3DES_DEC_CIPHER 4 +%define %%NUM_DES_BLOCKS %1 +%define %%DES_KS1 %2 +%define %%DES_KS2 %3 +%define %%DES_KS3 %4 + +%assign RN 0 +%assign LN 1 +%rep %%NUM_DES_BLOCKS + vmovdqa64 ZTMP12, ZW %+ RN ; keep R0 as IV for the next round + vmovdqa64 ZTMP13, ZW %+ LN ; keep L0 as IV for the next round + ;; DEC + DES_ENC_DEC DEC, ZW %+ RN, ZW %+ LN, %%DES_KS1, ZTMP0, ZTMP1, ZTMP2, ZTMP3, ZTMP4, ZTMP5, ZTMP6, ZTMP7, ZTMP8, ZTMP9, ZTMP10, ZTMP11 + ;; ENC + DES_ENC_DEC ENC, ZW %+ LN, ZW %+ RN, %%DES_KS2, ZTMP0, ZTMP1, ZTMP2, ZTMP3, ZTMP4, ZTMP5, ZTMP6, ZTMP7, ZTMP8, ZTMP9, ZTMP10, ZTMP11 + ;; DEC + DES_ENC_DEC DEC, ZW %+ RN, ZW %+ LN, %%DES_KS3, ZTMP0, ZTMP1, ZTMP2, ZTMP3, ZTMP4, ZTMP5, ZTMP6, ZTMP7, ZTMP8, ZTMP9, ZTMP10, ZTMP11 + vpxord ZW %+ RN, ZW %+ RN, ZIV1 ; R0 = R0 ^ IV1 + vpxord ZW %+ LN, ZW %+ LN, ZIV0 ; L0 = L0 ^ IV0 + vmovdqa64 ZIV0, ZTMP12 + vmovdqa64 ZIV1, ZTMP13 + +%assign RN (RN + 2) +%assign LN (LN + 2) +%endrep + +%endmacro + +;;; =========================================================================== +;;; DES CBC / DOCSIS DES ENCRYPT +;;; =========================================================================== +;;; +;;; DES_DOCSIS [in] - select between DES (DES CBC), DOCSIS (DOCSIS DES) and +;;; 3DES (3DES CBC) +;;; +;;; NOTE: clobbers OpMask registers +%macro GENERIC_DES_ENC 1 +%define %%DES_DOCSIS %1 + + ;; push the registers and allocate the stack frame + mov rax, rsp + sub rsp, STACKFRAME_size + and rsp, -64 + mov [rsp + _rsp_save], rax ; original SP + mov [rsp + _gpr_save + 0*8], r12 + mov [rsp + _gpr_save + 1*8], r13 + mov [rsp + _gpr_save + 2*8], r14 + mov [rsp + _gpr_save + 3*8], r15 + +%ifnidn %%DES_DOCSIS, 3DES + ;; DES and DOCSIS DES + DES_INIT STATE + _des_args_keys, STATE + _des_args_IV, rsp + _key_sched, ZIV0, ZIV1, ZW0, ZW1, ZW2, ZW3, ZW4, ZW5, ZW6, ZW7, ZW8, ZW9, ZW10, ZW11, ZW12, ZW13, ZW14, ZW15, ZTMP0, ZTMP1, ZTMP2, ZTMP3, ZTMP4, ZTMP5, ZTMP6, ZTMP7, ZTMP8, ZTMP9, ZTMP10, ZTMP11 +%else + ;; 3DES + DES3_INIT STATE + _des_args_keys, STATE + _des_args_IV, rsp + _key_sched, rsp + _key_sched2, rsp + _key_sched3, ZIV0, ZIV1, ZW0, ZW1, ZW2, ZW3, ZW4, ZW5, ZW6, ZW7, ZW8, ZW9, ZW10, ZW11, ZW12, ZW13, ZW14, ZW15, ZTMP0, ZTMP1, ZTMP2, ZTMP3, ZTMP4, ZTMP5, ZTMP6, ZTMP7, ZTMP8, ZTMP9, ZTMP10, ZTMP11, ENC +%endif + mov [rsp + _size_save], SIZE + and SIZE, -64 + xor OFFSET, OFFSET + ;; This loop processes message in blocks of 64 bytes. + ;; Anything smaller than 64 bytes is handled separately after the loop. +%%_gen_des_enc_loop: + cmp OFFSET, SIZE + jz %%_gen_des_enc_loop_end + ;; run loads + mov IA0, [STATE + _des_args_in + (0*PTR_SZ)] + mov IA1, [STATE + _des_args_in + (1*PTR_SZ)] + mov IA2, [STATE + _des_args_in + (2*PTR_SZ)] + mov INP0, [STATE + _des_args_in + (3*PTR_SZ)] + mov INP1, [STATE + _des_args_in + (4*PTR_SZ)] + mov INP2, [STATE + _des_args_in + (5*PTR_SZ)] + mov INP3, [STATE + _des_args_in + (6*PTR_SZ)] + mov INP4, [STATE + _des_args_in + (7*PTR_SZ)] + vmovdqu64 ZW0, [IA0 + OFFSET] + vmovdqu64 ZW1, [IA1 + OFFSET] + vmovdqu64 ZW2, [IA2 + OFFSET] + vmovdqu64 ZW3, [INP0 + OFFSET] + vmovdqu64 ZW4, [INP1 + OFFSET] + vmovdqu64 ZW5, [INP2 + OFFSET] + vmovdqu64 ZW6, [INP3 + OFFSET] + vmovdqu64 ZW7, [INP4 + OFFSET] + + mov IA0, [STATE + _des_args_in + (8*PTR_SZ)] + mov IA1, [STATE + _des_args_in + (9*PTR_SZ)] + mov IA2, [STATE + _des_args_in + (10*PTR_SZ)] + mov INP0, [STATE + _des_args_in + (11*PTR_SZ)] + mov INP1, [STATE + _des_args_in + (12*PTR_SZ)] + mov INP2, [STATE + _des_args_in + (13*PTR_SZ)] + mov INP3, [STATE + _des_args_in + (14*PTR_SZ)] + mov INP4, [STATE + _des_args_in + (15*PTR_SZ)] + vmovdqu64 ZW8, [IA0 + OFFSET] + vmovdqu64 ZW9, [IA1 + OFFSET] + vmovdqu64 ZW10, [IA2 + OFFSET] + vmovdqu64 ZW11, [INP0 + OFFSET] + vmovdqu64 ZW12, [INP1 + OFFSET] + vmovdqu64 ZW13, [INP2 + OFFSET] + vmovdqu64 ZW14, [INP3 + OFFSET] + vmovdqu64 ZW15, [INP4 + OFFSET] + + ;; Transpose input + TRANSPOSE_IN ZW0, ZW1, ZW2, ZW3, ZW4, ZW5, ZW6, ZW7, ZW8, ZW9, ZW10, ZW11, ZW12, ZW13, ZW14, ZW15, ZTMP0, ZTMP1, ZTMP2, ZTMP3, ZTMP4, ZTMP5, ZTMP6, ZTMP7, ZTMP8, ZTMP9, ZTMP10, ZTMP11, ZTMP12, ZTMP13 + + ;; DES CBC ENC comes here + vpxord ZW0, ZW0, ZIV0 ; R0 = R0 ^ IV0 + vpxord ZW1, ZW1, ZIV1 ; L0 = L0 ^ IV1 + +%ifnidn %%DES_DOCSIS, 3DES + GEN_DES_ENC_CIPHER 8, rsp + _key_sched +%else + GEN_3DES_ENC_CIPHER 8, rsp + _key_sched, rsp + _key_sched2, rsp + _key_sched3 +%endif + + ;; transpose data on output + TRANSPOSE_OUT ZW0, ZW1, ZW2, ZW3, ZW4, ZW5, ZW6, ZW7, ZW8, ZW9, ZW10, ZW11, ZW12, ZW13, ZW14, ZW15, ZTMP0, ZTMP1, ZTMP2, ZTMP3, ZTMP4, ZTMP5, ZTMP6, ZTMP7, ZTMP8, ZTMP9, ZTMP10, ZTMP11, ZTMP12, ZTMP13 + ;; run stores + mov IA0, [STATE + _des_args_out + (0*PTR_SZ)] + mov IA1, [STATE + _des_args_out + (1*PTR_SZ)] + mov IA2, [STATE + _des_args_out + (2*PTR_SZ)] + mov INP0, [STATE + _des_args_out + (3*PTR_SZ)] + mov INP1, [STATE + _des_args_out + (4*PTR_SZ)] + mov INP2, [STATE + _des_args_out + (5*PTR_SZ)] + mov INP3, [STATE + _des_args_out + (6*PTR_SZ)] + mov INP4, [STATE + _des_args_out + (7*PTR_SZ)] + vmovdqu64 [IA0 + OFFSET], ZW0 + vmovdqu64 [IA1 + OFFSET], ZW1 + vmovdqu64 [IA2 + OFFSET], ZW2 + vmovdqu64 [INP0 + OFFSET], ZW3 + vmovdqu64 [INP1 + OFFSET], ZW4 + vmovdqu64 [INP2 + OFFSET], ZW5 + vmovdqu64 [INP3 + OFFSET], ZW6 + vmovdqu64 [INP4 + OFFSET], ZW7 + + mov IA0, [STATE + _des_args_out + (8*PTR_SZ)] + mov IA1, [STATE + _des_args_out + (9*PTR_SZ)] + mov IA2, [STATE + _des_args_out + (10*PTR_SZ)] + mov INP0, [STATE + _des_args_out + (11*PTR_SZ)] + mov INP1, [STATE + _des_args_out + (12*PTR_SZ)] + mov INP2, [STATE + _des_args_out + (13*PTR_SZ)] + mov INP3, [STATE + _des_args_out + (14*PTR_SZ)] + mov INP4, [STATE + _des_args_out + (15*PTR_SZ)] + vmovdqu64 [IA0 + OFFSET], ZW8 + vmovdqu64 [IA1 + OFFSET], ZW9 + vmovdqu64 [IA2 + OFFSET], ZW10 + vmovdqu64 [INP0 + OFFSET], ZW11 + vmovdqu64 [INP1 + OFFSET], ZW12 + vmovdqu64 [INP2 + OFFSET], ZW13 + vmovdqu64 [INP3 + OFFSET], ZW14 + vmovdqu64 [INP4 + OFFSET], ZW15 + + add OFFSET, 64 + jmp %%_gen_des_enc_loop +%%_gen_des_enc_loop_end: + ;; This is where we check if there is anything less than 64 bytes + ;; of message left for processing. + mov SIZE, [rsp + _size_save] + cmp OFFSET, SIZE + jz %%_gen_des_enc_part_end + ;; calculate min of bytes_left and 64, convert to qword mask + GET_MASK8 IA0 ; IA0 = mask + + kmovw k7, DWORD(IA0) + mov [rsp + _mask_save], IA0 + ;; run masked loads + mov IA0, [STATE + _des_args_in + (0*PTR_SZ)] + mov IA1, [STATE + _des_args_in + (1*PTR_SZ)] + mov IA2, [STATE + _des_args_in + (2*PTR_SZ)] + mov INP0, [STATE + _des_args_in + (3*PTR_SZ)] + mov INP1, [STATE + _des_args_in + (4*PTR_SZ)] + mov INP2, [STATE + _des_args_in + (5*PTR_SZ)] + mov INP3, [STATE + _des_args_in + (6*PTR_SZ)] + mov INP4, [STATE + _des_args_in + (7*PTR_SZ)] + vmovdqu64 ZW0{k7}{z}, [IA0 + OFFSET] + vmovdqu64 ZW1{k7}{z}, [IA1 + OFFSET] + vmovdqu64 ZW2{k7}{z}, [IA2 + OFFSET] + vmovdqu64 ZW3{k7}{z}, [INP0 + OFFSET] + vmovdqu64 ZW4{k7}{z}, [INP1 + OFFSET] + vmovdqu64 ZW5{k7}{z}, [INP2 + OFFSET] + vmovdqu64 ZW6{k7}{z}, [INP3 + OFFSET] + vmovdqu64 ZW7{k7}{z}, [INP4 + OFFSET] + + mov IA0, [STATE + _des_args_in + (8*PTR_SZ)] + mov IA1, [STATE + _des_args_in + (9*PTR_SZ)] + mov IA2, [STATE + _des_args_in + (10*PTR_SZ)] + mov INP0, [STATE + _des_args_in + (11*PTR_SZ)] + mov INP1, [STATE + _des_args_in + (12*PTR_SZ)] + mov INP2, [STATE + _des_args_in + (13*PTR_SZ)] + mov INP3, [STATE + _des_args_in + (14*PTR_SZ)] + mov INP4, [STATE + _des_args_in + (15*PTR_SZ)] + vmovdqu64 ZW8{k7}{z}, [IA0 + OFFSET] + vmovdqu64 ZW9{k7}{z}, [IA1 + OFFSET] + vmovdqu64 ZW10{k7}{z}, [IA2 + OFFSET] + vmovdqu64 ZW11{k7}{z}, [INP0 + OFFSET] + vmovdqu64 ZW12{k7}{z}, [INP1 + OFFSET] + vmovdqu64 ZW13{k7}{z}, [INP2 + OFFSET] + vmovdqu64 ZW14{k7}{z}, [INP3 + OFFSET] + vmovdqu64 ZW15{k7}{z}, [INP4 + OFFSET] + + ;; Transpose input + TRANSPOSE_IN ZW0, ZW1, ZW2, ZW3, ZW4, ZW5, ZW6, ZW7, ZW8, ZW9, ZW10, ZW11, ZW12, ZW13, ZW14, ZW15, ZTMP0, ZTMP1, ZTMP2, ZTMP3, ZTMP4, ZTMP5, ZTMP6, ZTMP7, ZTMP8, ZTMP9, ZTMP10, ZTMP11, ZTMP12, ZTMP13 + + ;; DES CBC ENC comes here + vpxord ZW0, ZW0, ZIV0 ; R0 = R0 ^ IV0 + vpxord ZW1, ZW1, ZIV1 ; L0 = L0 ^ IV1 + + mov IA0, [rsp + _mask_save] + cmp BYTE(IA0), 0x0f + ja %%_gt_4 + jz %%_blocks_4 + + cmp BYTE(IA0), 0x03 + ja %%_blocks_3 + jz %%_blocks_2 + + ;; process one block and move to transpose out +%ifnidn %%DES_DOCSIS, 3DES + GEN_DES_ENC_CIPHER 1, rsp + _key_sched +%else + GEN_3DES_ENC_CIPHER 1, rsp + _key_sched, rsp + _key_sched2, rsp + _key_sched3 +%endif + jmp %%_transpose_out + +%%_blocks_2: + ;; process two blocks and move to transpose out +%ifnidn %%DES_DOCSIS, 3DES + GEN_DES_ENC_CIPHER 2, rsp + _key_sched +%else + GEN_3DES_ENC_CIPHER 2, rsp + _key_sched, rsp + _key_sched2, rsp + _key_sched3 +%endif + jmp %%_transpose_out + +%%_blocks_3: + ;; process three blocks and move to transpose out +%ifnidn %%DES_DOCSIS, 3DES + GEN_DES_ENC_CIPHER 3, rsp + _key_sched +%else + GEN_3DES_ENC_CIPHER 3, rsp + _key_sched, rsp + _key_sched2, rsp + _key_sched3 +%endif + jmp %%_transpose_out + +%%_blocks_4: + ;; process four blocks and move to transpose out +%ifnidn %%DES_DOCSIS, 3DES + GEN_DES_ENC_CIPHER 4, rsp + _key_sched +%else + GEN_3DES_ENC_CIPHER 4, rsp + _key_sched, rsp + _key_sched2, rsp + _key_sched3 +%endif + jmp %%_transpose_out + +%%_gt_4: + cmp BYTE(IA0), 0x3f + ja %%_blocks_7 + jz %%_blocks_6 +%%_blocks_5: + ;; process five blocks and move to transpose out +%ifnidn %%DES_DOCSIS, 3DES + GEN_DES_ENC_CIPHER 5, rsp + _key_sched +%else + GEN_3DES_ENC_CIPHER 5, rsp + _key_sched, rsp + _key_sched2, rsp + _key_sched3 +%endif + jmp %%_transpose_out + +%%_blocks_6: + ;; process six blocks and move to transpose out +%ifnidn %%DES_DOCSIS, 3DES + GEN_DES_ENC_CIPHER 6, rsp + _key_sched +%else + GEN_3DES_ENC_CIPHER 6, rsp + _key_sched, rsp + _key_sched2, rsp + _key_sched3 +%endif + jmp %%_transpose_out + +%%_blocks_7: + ;; process seven blocks and move to transpose out +%ifnidn %%DES_DOCSIS, 3DES + GEN_DES_ENC_CIPHER 7, rsp + _key_sched +%else + GEN_3DES_ENC_CIPHER 7, rsp + _key_sched, rsp + _key_sched2, rsp + _key_sched3 +%endif + +%%_transpose_out: + ;; transpose data on output + TRANSPOSE_OUT ZW0, ZW1, ZW2, ZW3, ZW4, ZW5, ZW6, ZW7, ZW8, ZW9, ZW10, ZW11, ZW12, ZW13, ZW14, ZW15, ZTMP0, ZTMP1, ZTMP2, ZTMP3, ZTMP4, ZTMP5, ZTMP6, ZTMP7, ZTMP8, ZTMP9, ZTMP10, ZTMP11, ZTMP12, ZTMP13 + + ;; run masked stores + mov IA0, [STATE + _des_args_out + (0*PTR_SZ)] + mov IA1, [STATE + _des_args_out + (1*PTR_SZ)] + mov IA2, [STATE + _des_args_out + (2*PTR_SZ)] + mov INP0, [STATE + _des_args_out + (3*PTR_SZ)] + mov INP1, [STATE + _des_args_out + (4*PTR_SZ)] + mov INP2, [STATE + _des_args_out + (5*PTR_SZ)] + mov INP3, [STATE + _des_args_out + (6*PTR_SZ)] + mov INP4, [STATE + _des_args_out + (7*PTR_SZ)] + vmovdqu64 [IA0 + OFFSET]{k7}, ZW0 + vmovdqu64 [IA1 + OFFSET]{k7}, ZW1 + vmovdqu64 [IA2 + OFFSET]{k7}, ZW2 + vmovdqu64 [INP0 + OFFSET]{k7}, ZW3 + vmovdqu64 [INP1 + OFFSET]{k7}, ZW4 + vmovdqu64 [INP2 + OFFSET]{k7}, ZW5 + vmovdqu64 [INP3 + OFFSET]{k7}, ZW6 + vmovdqu64 [INP4 + OFFSET]{k7}, ZW7 + + mov IA0, [STATE + _des_args_out + (8*PTR_SZ)] + mov IA1, [STATE + _des_args_out + (9*PTR_SZ)] + mov IA2, [STATE + _des_args_out + (10*PTR_SZ)] + mov INP0, [STATE + _des_args_out + (11*PTR_SZ)] + mov INP1, [STATE + _des_args_out + (12*PTR_SZ)] + mov INP2, [STATE + _des_args_out + (13*PTR_SZ)] + mov INP3, [STATE + _des_args_out + (14*PTR_SZ)] + mov INP4, [STATE + _des_args_out + (15*PTR_SZ)] + vmovdqu64 [IA0 + OFFSET]{k7}, ZW8 + vmovdqu64 [IA1 + OFFSET]{k7}, ZW9 + vmovdqu64 [IA2 + OFFSET]{k7}, ZW10 + vmovdqu64 [INP0 + OFFSET]{k7}, ZW11 + vmovdqu64 [INP1 + OFFSET]{k7}, ZW12 + vmovdqu64 [INP2 + OFFSET]{k7}, ZW13 + vmovdqu64 [INP3 + OFFSET]{k7}, ZW14 + vmovdqu64 [INP4 + OFFSET]{k7}, ZW15 +%%_gen_des_enc_part_end: + + ;; store IV and update pointers + DES_FINISH ZIV0, ZIV1, ZTMP0, ZTMP1, ZTMP2, ZTMP3, ZTMP4 + + ;; CFB part for DOCSIS +%ifidn %%DES_DOCSIS, DOCSIS + DES_CFB_ONE ENC, rsp + _key_sched, ZW0, ZW1, ZW2, ZW3, ZW4, ZW5, ZW6, ZW7, ZW8, ZW9, ZW10, ZW11, ZW12, ZW13, ZW14, ZW15, ZTMP0, ZTMP1, ZTMP2, ZTMP3, ZTMP4, ZTMP5, ZTMP6, ZTMP7, ZTMP8, rsp + _tmp_in, rsp + _tmp_out, rsp + _tmp_iv, rsp + _tmp_mask +%endif + + CLEAR_KEY_SCHEDULE %%DES_DOCSIS, ZW0 + + ;; restore stack pointer and registers + mov r12, [rsp + _gpr_save + 0*8] + mov r13, [rsp + _gpr_save + 1*8] + mov r14, [rsp + _gpr_save + 2*8] + mov r15, [rsp + _gpr_save + 3*8] + mov rsp, [rsp + _rsp_save] ; original SP +%endmacro + +;;; =========================================================================== +;;; DES CBC / DOCSIS DES DECRYPT +;;; =========================================================================== +;;; +;;; DES_DOCSIS [in] - select between DES (DES CBC), DOCSIS (DOCSIS DES) and +;;; 3DES (3DES CBC) +;;; +;;; NOTE: clobbers OpMask registers +%macro GENERIC_DES_DEC 1 +%define %%DES_DOCSIS %1 + + ;; push the registers and allocate the stack frame + mov rax, rsp + sub rsp, STACKFRAME_size + and rsp, -64 + mov [rsp + _rsp_save], rax ; original SP + mov [rsp + _gpr_save + 0*8], r12 + mov [rsp + _gpr_save + 1*8], r13 + mov [rsp + _gpr_save + 2*8], r14 + mov [rsp + _gpr_save + 3*8], r15 + +%ifnidn %%DES_DOCSIS, 3DES + ;; DES and DOCSIS + DES_INIT STATE + _des_args_keys, STATE + _des_args_IV, rsp + _key_sched, ZIV0, ZIV1, ZW0, ZW1, ZW2, ZW3, ZW4, ZW5, ZW6, ZW7, ZW8, ZW9, ZW10, ZW11, ZW12, ZW13, ZW14, ZW15, ZTMP0, ZTMP1, ZTMP2, ZTMP3, ZTMP4, ZTMP5, ZTMP6, ZTMP7, ZTMP8, ZTMP9, ZTMP10, ZTMP11 +%else + ;; 3DES + DES3_INIT STATE + _des_args_keys, STATE + _des_args_IV, rsp + _key_sched, rsp + _key_sched2, rsp + _key_sched3, ZIV0, ZIV1, ZW0, ZW1, ZW2, ZW3, ZW4, ZW5, ZW6, ZW7, ZW8, ZW9, ZW10, ZW11, ZW12, ZW13, ZW14, ZW15, ZTMP0, ZTMP1, ZTMP2, ZTMP3, ZTMP4, ZTMP5, ZTMP6, ZTMP7, ZTMP8, ZTMP9, ZTMP10, ZTMP11, DEC +%endif + + ;; CFB part for DOCSIS +%ifidn %%DES_DOCSIS, DOCSIS + DES_CFB_ONE DEC, rsp + _key_sched, ZW0, ZW1, ZW2, ZW3, ZW4, ZW5, ZW6, ZW7, ZW8, ZW9, ZW10, ZW11, ZW12, ZW13, ZW14, ZW15, ZTMP0, ZTMP1, ZTMP2, ZTMP3, ZTMP4, ZTMP5, ZTMP6, ZTMP7, ZTMP8, rsp + _tmp_in, rsp + _tmp_out, rsp + _tmp_iv, rsp + _tmp_mask +%endif + + mov [rsp + _size_save], SIZE + and SIZE, -64 + xor OFFSET, OFFSET + ;; This loop processes message in blocks of 64 bytes. + ;; Anything smaller than 64 bytes is handled separately after the loop. +%%_gen_des_dec_loop: + cmp OFFSET, SIZE + jz %%_gen_des_dec_loop_end + ;; run loads + mov IA0, [STATE + _des_args_in + (0*PTR_SZ)] + mov IA1, [STATE + _des_args_in + (1*PTR_SZ)] + mov IA2, [STATE + _des_args_in + (2*PTR_SZ)] + mov INP0, [STATE + _des_args_in + (3*PTR_SZ)] + mov INP1, [STATE + _des_args_in + (4*PTR_SZ)] + mov INP2, [STATE + _des_args_in + (5*PTR_SZ)] + mov INP3, [STATE + _des_args_in + (6*PTR_SZ)] + mov INP4, [STATE + _des_args_in + (7*PTR_SZ)] + vmovdqu64 ZW0, [IA0 + OFFSET] + vmovdqu64 ZW1, [IA1 + OFFSET] + vmovdqu64 ZW2, [IA2 + OFFSET] + vmovdqu64 ZW3, [INP0 + OFFSET] + vmovdqu64 ZW4, [INP1 + OFFSET] + vmovdqu64 ZW5, [INP2 + OFFSET] + vmovdqu64 ZW6, [INP3 + OFFSET] + vmovdqu64 ZW7, [INP4 + OFFSET] + + mov IA0, [STATE + _des_args_in + (8*PTR_SZ)] + mov IA1, [STATE + _des_args_in + (9*PTR_SZ)] + mov IA2, [STATE + _des_args_in + (10*PTR_SZ)] + mov INP0, [STATE + _des_args_in + (11*PTR_SZ)] + mov INP1, [STATE + _des_args_in + (12*PTR_SZ)] + mov INP2, [STATE + _des_args_in + (13*PTR_SZ)] + mov INP3, [STATE + _des_args_in + (14*PTR_SZ)] + mov INP4, [STATE + _des_args_in + (15*PTR_SZ)] + vmovdqu64 ZW8, [IA0 + OFFSET] + vmovdqu64 ZW9, [IA1 + OFFSET] + vmovdqu64 ZW10, [IA2 + OFFSET] + vmovdqu64 ZW11, [INP0 + OFFSET] + vmovdqu64 ZW12, [INP1 + OFFSET] + vmovdqu64 ZW13, [INP2 + OFFSET] + vmovdqu64 ZW14, [INP3 + OFFSET] + vmovdqu64 ZW15, [INP4 + OFFSET] + + ;; Transpose input + TRANSPOSE_IN ZW0, ZW1, ZW2, ZW3, ZW4, ZW5, ZW6, ZW7, ZW8, ZW9, ZW10, ZW11, ZW12, ZW13, ZW14, ZW15, ZTMP0, ZTMP1, ZTMP2, ZTMP3, ZTMP4, ZTMP5, ZTMP6, ZTMP7, ZTMP8, ZTMP9, ZTMP10, ZTMP11, ZTMP12, ZTMP13 + +%ifnidn %%DES_DOCSIS, 3DES + ;; DES CBC DEC comes here + GEN_DES_DEC_CIPHER 8, rsp + _key_sched +%else + ;; 3DES CBC DEC comes here + GEN_3DES_DEC_CIPHER 8, rsp + _key_sched, rsp + _key_sched2, rsp + _key_sched3 +%endif + + ;; transpose data on output + TRANSPOSE_OUT ZW0, ZW1, ZW2, ZW3, ZW4, ZW5, ZW6, ZW7, ZW8, ZW9, ZW10, ZW11, ZW12, ZW13, ZW14, ZW15, ZTMP0, ZTMP1, ZTMP2, ZTMP3, ZTMP4, ZTMP5, ZTMP6, ZTMP7, ZTMP8, ZTMP9, ZTMP10, ZTMP11, ZTMP12, ZTMP13 + + ;; run stores + mov IA0, [STATE + _des_args_out + (0*PTR_SZ)] + mov IA1, [STATE + _des_args_out + (1*PTR_SZ)] + mov IA2, [STATE + _des_args_out + (2*PTR_SZ)] + mov INP0, [STATE + _des_args_out + (3*PTR_SZ)] + mov INP1, [STATE + _des_args_out + (4*PTR_SZ)] + mov INP2, [STATE + _des_args_out + (5*PTR_SZ)] + mov INP3, [STATE + _des_args_out + (6*PTR_SZ)] + mov INP4, [STATE + _des_args_out + (7*PTR_SZ)] + vmovdqu64 [IA0 + OFFSET], ZW0 + vmovdqu64 [IA1 + OFFSET], ZW1 + vmovdqu64 [IA2 + OFFSET], ZW2 + vmovdqu64 [INP0 + OFFSET], ZW3 + vmovdqu64 [INP1 + OFFSET], ZW4 + vmovdqu64 [INP2 + OFFSET], ZW5 + vmovdqu64 [INP3 + OFFSET], ZW6 + vmovdqu64 [INP4 + OFFSET], ZW7 + + mov IA0, [STATE + _des_args_out + (8*PTR_SZ)] + mov IA1, [STATE + _des_args_out + (9*PTR_SZ)] + mov IA2, [STATE + _des_args_out + (10*PTR_SZ)] + mov INP0, [STATE + _des_args_out + (11*PTR_SZ)] + mov INP1, [STATE + _des_args_out + (12*PTR_SZ)] + mov INP2, [STATE + _des_args_out + (13*PTR_SZ)] + mov INP3, [STATE + _des_args_out + (14*PTR_SZ)] + mov INP4, [STATE + _des_args_out + (15*PTR_SZ)] + vmovdqu64 [IA0 + OFFSET], ZW8 + vmovdqu64 [IA1 + OFFSET], ZW9 + vmovdqu64 [IA2 + OFFSET], ZW10 + vmovdqu64 [INP0 + OFFSET], ZW11 + vmovdqu64 [INP1 + OFFSET], ZW12 + vmovdqu64 [INP2 + OFFSET], ZW13 + vmovdqu64 [INP3 + OFFSET], ZW14 + vmovdqu64 [INP4 + OFFSET], ZW15 + + add OFFSET, 64 + jmp %%_gen_des_dec_loop +%%_gen_des_dec_loop_end: + ;; This is where we check if there is anything less than 64 bytes + ;; of message left for processing. + mov SIZE, [rsp + _size_save] + cmp OFFSET, SIZE + jz %%_gen_des_dec_part_end + ;; calculate min of bytes_left and 64, convert to qword mask + GET_MASK8 IA0 ; IA0 = mask + + kmovw k7, DWORD(IA0) + mov [rsp + _mask_save], IA0 + ;; run masked loads + mov IA0, [STATE + _des_args_in + (0*PTR_SZ)] + mov IA1, [STATE + _des_args_in + (1*PTR_SZ)] + mov IA2, [STATE + _des_args_in + (2*PTR_SZ)] + mov INP0, [STATE + _des_args_in + (3*PTR_SZ)] + mov INP1, [STATE + _des_args_in + (4*PTR_SZ)] + mov INP2, [STATE + _des_args_in + (5*PTR_SZ)] + mov INP3, [STATE + _des_args_in + (6*PTR_SZ)] + mov INP4, [STATE + _des_args_in + (7*PTR_SZ)] + vmovdqu64 ZW0{k7}{z}, [IA0 + OFFSET] + vmovdqu64 ZW1{k7}{z}, [IA1 + OFFSET] + vmovdqu64 ZW2{k7}{z}, [IA2 + OFFSET] + vmovdqu64 ZW3{k7}{z}, [INP0 + OFFSET] + vmovdqu64 ZW4{k7}{z}, [INP1 + OFFSET] + vmovdqu64 ZW5{k7}{z}, [INP2 + OFFSET] + vmovdqu64 ZW6{k7}{z}, [INP3 + OFFSET] + vmovdqu64 ZW7{k7}{z}, [INP4 + OFFSET] + + mov IA0, [STATE + _des_args_in + (8*PTR_SZ)] + mov IA1, [STATE + _des_args_in + (9*PTR_SZ)] + mov IA2, [STATE + _des_args_in + (10*PTR_SZ)] + mov INP0, [STATE + _des_args_in + (11*PTR_SZ)] + mov INP1, [STATE + _des_args_in + (12*PTR_SZ)] + mov INP2, [STATE + _des_args_in + (13*PTR_SZ)] + mov INP3, [STATE + _des_args_in + (14*PTR_SZ)] + mov INP4, [STATE + _des_args_in + (15*PTR_SZ)] + vmovdqu64 ZW8{k7}{z}, [IA0 + OFFSET] + vmovdqu64 ZW9{k7}{z}, [IA1 + OFFSET] + vmovdqu64 ZW10{k7}{z}, [IA2 + OFFSET] + vmovdqu64 ZW11{k7}{z}, [INP0 + OFFSET] + vmovdqu64 ZW12{k7}{z}, [INP1 + OFFSET] + vmovdqu64 ZW13{k7}{z}, [INP2 + OFFSET] + vmovdqu64 ZW14{k7}{z}, [INP3 + OFFSET] + vmovdqu64 ZW15{k7}{z}, [INP4 + OFFSET] + + ;; Transpose input + TRANSPOSE_IN ZW0, ZW1, ZW2, ZW3, ZW4, ZW5, ZW6, ZW7, ZW8, ZW9, ZW10, ZW11, ZW12, ZW13, ZW14, ZW15, ZTMP0, ZTMP1, ZTMP2, ZTMP3, ZTMP4, ZTMP5, ZTMP6, ZTMP7, ZTMP8, ZTMP9, ZTMP10, ZTMP11, ZTMP12, ZTMP13 + + ;; DES CBC DEC comes here + mov IA0, [rsp + _mask_save] + cmp BYTE(IA0), 0x0f + ja %%_gt_4 + jz %%_blocks_4 + + cmp BYTE(IA0), 0x03 + ja %%_blocks_3 + jz %%_blocks_2 + ;; process one block and move to transpose out +%ifnidn %%DES_DOCSIS, 3DES + GEN_DES_DEC_CIPHER 1, rsp + _key_sched +%else + GEN_3DES_DEC_CIPHER 1, rsp + _key_sched, rsp + _key_sched2, rsp + _key_sched3 +%endif + jmp %%_transpose_out + +%%_blocks_2: + ;; process two blocks and move to transpose out +%ifnidn %%DES_DOCSIS, 3DES + GEN_DES_DEC_CIPHER 2, rsp + _key_sched +%else + GEN_3DES_DEC_CIPHER 2, rsp + _key_sched, rsp + _key_sched2, rsp + _key_sched3 +%endif + jmp %%_transpose_out + +%%_blocks_3: + ;; process three blocks and move to transpose out +%ifnidn %%DES_DOCSIS, 3DES + GEN_DES_DEC_CIPHER 3, rsp + _key_sched +%else + GEN_3DES_DEC_CIPHER 3, rsp + _key_sched, rsp + _key_sched2, rsp + _key_sched3 +%endif + jmp %%_transpose_out + +%%_blocks_4: + ;; process four blocks and move to transpose out +%ifnidn %%DES_DOCSIS, 3DES + GEN_DES_DEC_CIPHER 4, rsp + _key_sched +%else + GEN_3DES_DEC_CIPHER 4, rsp + _key_sched, rsp + _key_sched2, rsp + _key_sched3 +%endif + jmp %%_transpose_out + +%%_gt_4: + cmp BYTE(IA0), 0x3f + ja %%_blocks_7 + jz %%_blocks_6 +%%_blocks_5: + ;; process five blocks and move to transpose out +%ifnidn %%DES_DOCSIS, 3DES + GEN_DES_DEC_CIPHER 5, rsp + _key_sched +%else + GEN_3DES_DEC_CIPHER 5, rsp + _key_sched, rsp + _key_sched2, rsp + _key_sched3 +%endif + jmp %%_transpose_out + +%%_blocks_6: + ;; process six blocks and move to transpose out +%ifnidn %%DES_DOCSIS, 3DES + GEN_DES_DEC_CIPHER 6, rsp + _key_sched +%else + GEN_3DES_DEC_CIPHER 6, rsp + _key_sched, rsp + _key_sched2, rsp + _key_sched3 +%endif + jmp %%_transpose_out + +%%_blocks_7: + ;; process seven blocks and move to transpose out +%ifnidn %%DES_DOCSIS, 3DES + GEN_DES_DEC_CIPHER 7, rsp + _key_sched +%else + GEN_3DES_DEC_CIPHER 7, rsp + _key_sched, rsp + _key_sched2, rsp + _key_sched3 +%endif + +%%_transpose_out: + ;; transpose data on output + TRANSPOSE_OUT ZW0, ZW1, ZW2, ZW3, ZW4, ZW5, ZW6, ZW7, ZW8, ZW9, ZW10, ZW11, ZW12, ZW13, ZW14, ZW15, ZTMP0, ZTMP1, ZTMP2, ZTMP3, ZTMP4, ZTMP5, ZTMP6, ZTMP7, ZTMP8, ZTMP9, ZTMP10, ZTMP11, ZTMP12, ZTMP13 + + ;; run masked stores + mov IA0, [STATE + _des_args_out + (0*PTR_SZ)] + mov IA1, [STATE + _des_args_out + (1*PTR_SZ)] + mov IA2, [STATE + _des_args_out + (2*PTR_SZ)] + mov INP0, [STATE + _des_args_out + (3*PTR_SZ)] + mov INP1, [STATE + _des_args_out + (4*PTR_SZ)] + mov INP2, [STATE + _des_args_out + (5*PTR_SZ)] + mov INP3, [STATE + _des_args_out + (6*PTR_SZ)] + mov INP4, [STATE + _des_args_out + (7*PTR_SZ)] + vmovdqu64 [IA0 + OFFSET]{k7}, ZW0 + vmovdqu64 [IA1 + OFFSET]{k7}, ZW1 + vmovdqu64 [IA2 + OFFSET]{k7}, ZW2 + vmovdqu64 [INP0 + OFFSET]{k7}, ZW3 + vmovdqu64 [INP1 + OFFSET]{k7}, ZW4 + vmovdqu64 [INP2 + OFFSET]{k7}, ZW5 + vmovdqu64 [INP3 + OFFSET]{k7}, ZW6 + vmovdqu64 [INP4 + OFFSET]{k7}, ZW7 + + mov IA0, [STATE + _des_args_out + (8*PTR_SZ)] + mov IA1, [STATE + _des_args_out + (9*PTR_SZ)] + mov IA2, [STATE + _des_args_out + (10*PTR_SZ)] + mov INP0, [STATE + _des_args_out + (11*PTR_SZ)] + mov INP1, [STATE + _des_args_out + (12*PTR_SZ)] + mov INP2, [STATE + _des_args_out + (13*PTR_SZ)] + mov INP3, [STATE + _des_args_out + (14*PTR_SZ)] + mov INP4, [STATE + _des_args_out + (15*PTR_SZ)] + vmovdqu64 [IA0 + OFFSET]{k7}, ZW8 + vmovdqu64 [IA1 + OFFSET]{k7}, ZW9 + vmovdqu64 [IA2 + OFFSET]{k7}, ZW10 + vmovdqu64 [INP0 + OFFSET]{k7}, ZW11 + vmovdqu64 [INP1 + OFFSET]{k7}, ZW12 + vmovdqu64 [INP2 + OFFSET]{k7}, ZW13 + vmovdqu64 [INP3 + OFFSET]{k7}, ZW14 + vmovdqu64 [INP4 + OFFSET]{k7}, ZW15 +%%_gen_des_dec_part_end: + + ;; store IV and update pointers + DES_FINISH ZIV0, ZIV1, ZTMP0, ZTMP1, ZTMP2, ZTMP3, ZTMP4 + + CLEAR_KEY_SCHEDULE %%DES_DOCSIS, ZW0 + + ;; restore stack pointer and registers + mov r12, [rsp + _gpr_save + 0*8] + mov r13, [rsp + _gpr_save + 1*8] + mov r14, [rsp + _gpr_save + 2*8] + mov r15, [rsp + _gpr_save + 3*8] + mov rsp, [rsp + _rsp_save] ; original SP +%endmacro + + +;;; ======================================================== +;;; DATA + +section .data +default rel +align 64 +mask_values: + dd 0x04000000, 0x04000000, 0x04000000, 0x04000000 + dd 0x04000000, 0x04000000, 0x04000000, 0x04000000 + dd 0x04000000, 0x04000000, 0x04000000, 0x04000000 + dd 0x04000000, 0x04000000, 0x04000000, 0x04000000 + dd 0x40240202, 0x40240202, 0x40240202, 0x40240202 + dd 0x40240202, 0x40240202, 0x40240202, 0x40240202 + dd 0x40240202, 0x40240202, 0x40240202, 0x40240202 + dd 0x40240202, 0x40240202, 0x40240202, 0x40240202 + dd 0x00001110, 0x00001110, 0x00001110, 0x00001110 + dd 0x00001110, 0x00001110, 0x00001110, 0x00001110 + dd 0x00001110, 0x00001110, 0x00001110, 0x00001110 + dd 0x00001110, 0x00001110, 0x00001110, 0x00001110 + dd 0x01088000, 0x01088000, 0x01088000, 0x01088000 + dd 0x01088000, 0x01088000, 0x01088000, 0x01088000 + dd 0x01088000, 0x01088000, 0x01088000, 0x01088000 + dd 0x01088000, 0x01088000, 0x01088000, 0x01088000 + dd 0x00000001, 0x00000001, 0x00000001, 0x00000001 + dd 0x00000001, 0x00000001, 0x00000001, 0x00000001 + dd 0x00000001, 0x00000001, 0x00000001, 0x00000001 + dd 0x00000001, 0x00000001, 0x00000001, 0x00000001 + dd 0x0081000C, 0x0081000C, 0x0081000C, 0x0081000C + dd 0x0081000C, 0x0081000C, 0x0081000C, 0x0081000C + dd 0x0081000C, 0x0081000C, 0x0081000C, 0x0081000C + dd 0x0081000C, 0x0081000C, 0x0081000C, 0x0081000C + dd 0x00000020, 0x00000020, 0x00000020, 0x00000020 + dd 0x00000020, 0x00000020, 0x00000020, 0x00000020 + dd 0x00000020, 0x00000020, 0x00000020, 0x00000020 + dd 0x00000020, 0x00000020, 0x00000020, 0x00000020 + dd 0x00000040, 0x00000040, 0x00000040, 0x00000040 + dd 0x00000040, 0x00000040, 0x00000040, 0x00000040 + dd 0x00000040, 0x00000040, 0x00000040, 0x00000040 + dd 0x00000040, 0x00000040, 0x00000040, 0x00000040 + dd 0x00400400, 0x00400400, 0x00400400, 0x00400400 + dd 0x00400400, 0x00400400, 0x00400400, 0x00400400 + dd 0x00400400, 0x00400400, 0x00400400, 0x00400400 + dd 0x00400400, 0x00400400, 0x00400400, 0x00400400 + dd 0x00000800, 0x00000800, 0x00000800, 0x00000800 + dd 0x00000800, 0x00000800, 0x00000800, 0x00000800 + dd 0x00000800, 0x00000800, 0x00000800, 0x00000800 + dd 0x00000800, 0x00000800, 0x00000800, 0x00000800 + dd 0x00002000, 0x00002000, 0x00002000, 0x00002000 + dd 0x00002000, 0x00002000, 0x00002000, 0x00002000 + dd 0x00002000, 0x00002000, 0x00002000, 0x00002000 + dd 0x00002000, 0x00002000, 0x00002000, 0x00002000 + dd 0x00100000, 0x00100000, 0x00100000, 0x00100000 + dd 0x00100000, 0x00100000, 0x00100000, 0x00100000 + dd 0x00100000, 0x00100000, 0x00100000, 0x00100000 + dd 0x00100000, 0x00100000, 0x00100000, 0x00100000 + dd 0x00004000, 0x00004000, 0x00004000, 0x00004000 + dd 0x00004000, 0x00004000, 0x00004000, 0x00004000 + dd 0x00004000, 0x00004000, 0x00004000, 0x00004000 + dd 0x00004000, 0x00004000, 0x00004000, 0x00004000 + dd 0x00020000, 0x00020000, 0x00020000, 0x00020000 + dd 0x00020000, 0x00020000, 0x00020000, 0x00020000 + dd 0x00020000, 0x00020000, 0x00020000, 0x00020000 + dd 0x00020000, 0x00020000, 0x00020000, 0x00020000 + dd 0x02000000, 0x02000000, 0x02000000, 0x02000000 + dd 0x02000000, 0x02000000, 0x02000000, 0x02000000 + dd 0x02000000, 0x02000000, 0x02000000, 0x02000000 + dd 0x02000000, 0x02000000, 0x02000000, 0x02000000 + dd 0x08000000, 0x08000000, 0x08000000, 0x08000000 + dd 0x08000000, 0x08000000, 0x08000000, 0x08000000 + dd 0x08000000, 0x08000000, 0x08000000, 0x08000000 + dd 0x08000000, 0x08000000, 0x08000000, 0x08000000 + dd 0x00000080, 0x00000080, 0x00000080, 0x00000080 + dd 0x00000080, 0x00000080, 0x00000080, 0x00000080 + dd 0x00000080, 0x00000080, 0x00000080, 0x00000080 + dd 0x00000080, 0x00000080, 0x00000080, 0x00000080 + dd 0x20000000, 0x20000000, 0x20000000, 0x20000000 + dd 0x20000000, 0x20000000, 0x20000000, 0x20000000 + dd 0x20000000, 0x20000000, 0x20000000, 0x20000000 + dd 0x20000000, 0x20000000, 0x20000000, 0x20000000 + dd 0x90000000, 0x90000000, 0x90000000, 0x90000000 + dd 0x90000000, 0x90000000, 0x90000000, 0x90000000 + dd 0x90000000, 0x90000000, 0x90000000, 0x90000000 + dd 0x90000000, 0x90000000, 0x90000000, 0x90000000 + +align 64 +init_perm_consts: + dd 0x0f0f0f0f, 0x0f0f0f0f, 0x0f0f0f0f, 0x0f0f0f0f + dd 0x0f0f0f0f, 0x0f0f0f0f, 0x0f0f0f0f, 0x0f0f0f0f + dd 0x0f0f0f0f, 0x0f0f0f0f, 0x0f0f0f0f, 0x0f0f0f0f + dd 0x0f0f0f0f, 0x0f0f0f0f, 0x0f0f0f0f, 0x0f0f0f0f + dd 0x0000ffff, 0x0000ffff, 0x0000ffff, 0x0000ffff + dd 0x0000ffff, 0x0000ffff, 0x0000ffff, 0x0000ffff + dd 0x0000ffff, 0x0000ffff, 0x0000ffff, 0x0000ffff + dd 0x0000ffff, 0x0000ffff, 0x0000ffff, 0x0000ffff + dd 0x33333333, 0x33333333, 0x33333333, 0x33333333 + dd 0x33333333, 0x33333333, 0x33333333, 0x33333333 + dd 0x33333333, 0x33333333, 0x33333333, 0x33333333 + dd 0x33333333, 0x33333333, 0x33333333, 0x33333333 + dd 0x00ff00ff, 0x00ff00ff, 0x00ff00ff, 0x00ff00ff + dd 0x00ff00ff, 0x00ff00ff, 0x00ff00ff, 0x00ff00ff + dd 0x00ff00ff, 0x00ff00ff, 0x00ff00ff, 0x00ff00ff + dd 0x00ff00ff, 0x00ff00ff, 0x00ff00ff, 0x00ff00ff + dd 0x55555555, 0x55555555, 0x55555555, 0x55555555 + dd 0x55555555, 0x55555555, 0x55555555, 0x55555555 + dd 0x55555555, 0x55555555, 0x55555555, 0x55555555 + dd 0x55555555, 0x55555555, 0x55555555, 0x55555555 + +;;; S-Box table +align 64 +S_box_flipped: + ;; SBOX0 + dw 0x07, 0x02, 0x0c, 0x0f, 0x04, 0x0b, 0x0a, 0x0c + dw 0x0b, 0x07, 0x06, 0x09, 0x0d, 0x04, 0x00, 0x0a + dw 0x02, 0x08, 0x05, 0x03, 0x0f, 0x06, 0x09, 0x05 + dw 0x08, 0x01, 0x03, 0x0e, 0x01, 0x0d, 0x0e, 0x00 + dw 0x00, 0x0f, 0x05, 0x0a, 0x07, 0x02, 0x09, 0x05 + dw 0x0e, 0x01, 0x03, 0x0c, 0x0b, 0x08, 0x0c, 0x06 + dw 0x0f, 0x03, 0x06, 0x0d, 0x04, 0x09, 0x0a, 0x00 + dw 0x02, 0x04, 0x0d, 0x07, 0x08, 0x0e, 0x01, 0x0b + ;; SBOX1 + dw 0x0f, 0x00, 0x09, 0x0a, 0x06, 0x05, 0x03, 0x09 + dw 0x01, 0x0e, 0x04, 0x03, 0x0c, 0x0b, 0x0a, 0x04 + dw 0x08, 0x07, 0x0e, 0x01, 0x0d, 0x02, 0x00, 0x0c + dw 0x07, 0x0d, 0x0b, 0x06, 0x02, 0x08, 0x05, 0x0f + dw 0x0c, 0x0b, 0x03, 0x0d, 0x0f, 0x0c, 0x06, 0x00 + dw 0x02, 0x05, 0x08, 0x0e, 0x01, 0x02, 0x0d, 0x07 + dw 0x0b, 0x01, 0x00, 0x06, 0x04, 0x0f, 0x09, 0x0a + dw 0x0e, 0x08, 0x05, 0x03, 0x07, 0x04, 0x0a, 0x09 + ;; SBOX2 + dw 0x05, 0x0b, 0x08, 0x0d, 0x06, 0x01, 0x0d, 0x0a + dw 0x09, 0x02, 0x03, 0x04, 0x0f, 0x0c, 0x04, 0x07 + dw 0x00, 0x06, 0x0b, 0x08, 0x0c, 0x0f, 0x02, 0x05 + dw 0x07, 0x09, 0x0e, 0x03, 0x0a, 0x00, 0x01, 0x0e + dw 0x0b, 0x08, 0x04, 0x02, 0x0c, 0x06, 0x03, 0x0d + dw 0x00, 0x0b, 0x0a, 0x07, 0x06, 0x01, 0x0f, 0x04 + dw 0x0e, 0x05, 0x01, 0x0f, 0x02, 0x09, 0x0d, 0x0a + dw 0x09, 0x00, 0x07, 0x0c, 0x05, 0x0e, 0x08, 0x03 + ;; SBOX3 + dw 0x0e, 0x05, 0x08, 0x0f, 0x00, 0x03, 0x0d, 0x0a + dw 0x07, 0x09, 0x01, 0x0c, 0x09, 0x0e, 0x02, 0x01 + dw 0x0b, 0x06, 0x04, 0x08, 0x06, 0x0d, 0x03, 0x04 + dw 0x0c, 0x00, 0x0a, 0x07, 0x05, 0x0b, 0x0f, 0x02 + dw 0x0b, 0x0c, 0x02, 0x09, 0x06, 0x05, 0x08, 0x03 + dw 0x0d, 0x00, 0x04, 0x0a, 0x00, 0x0b, 0x07, 0x04 + dw 0x01, 0x0f, 0x0e, 0x02, 0x0f, 0x08, 0x05, 0x0e + dw 0x0a, 0x06, 0x03, 0x0d, 0x0c, 0x01, 0x09, 0x07 + ;; SBOX4 + dw 0x04, 0x02, 0x01, 0x0f, 0x0e, 0x05, 0x0b, 0x06 + dw 0x02, 0x08, 0x0c, 0x03, 0x0d, 0x0e, 0x07, 0x00 + dw 0x03, 0x04, 0x0a, 0x09, 0x05, 0x0b, 0x00, 0x0c + dw 0x08, 0x0d, 0x0f, 0x0a, 0x06, 0x01, 0x09, 0x07 + dw 0x07, 0x0d, 0x0a, 0x06, 0x02, 0x08, 0x0c, 0x05 + dw 0x04, 0x03, 0x0f, 0x00, 0x0b, 0x04, 0x01, 0x0a + dw 0x0d, 0x01, 0x00, 0x0f, 0x0e, 0x07, 0x09, 0x02 + dw 0x03, 0x0e, 0x05, 0x09, 0x08, 0x0b, 0x06, 0x0c + ;; SBOX5 + dw 0x03, 0x09, 0x00, 0x0e, 0x09, 0x04, 0x07, 0x08 + dw 0x05, 0x0f, 0x0c, 0x02, 0x06, 0x03, 0x0a, 0x0d + dw 0x08, 0x07, 0x0b, 0x00, 0x04, 0x01, 0x0e, 0x0b + dw 0x0f, 0x0a, 0x02, 0x05, 0x01, 0x0c, 0x0d, 0x06 + dw 0x05, 0x02, 0x06, 0x0d, 0x0e, 0x09, 0x00, 0x06 + dw 0x02, 0x04, 0x0b, 0x08, 0x09, 0x0f, 0x0c, 0x01 + dw 0x0f, 0x0c, 0x08, 0x07, 0x03, 0x0a, 0x0d, 0x00 + dw 0x04, 0x03, 0x07, 0x0e, 0x0a, 0x05, 0x01, 0x0b + ;; SBOX6 + dw 0x02, 0x08, 0x0c, 0x05, 0x0f, 0x03, 0x0a, 0x00 + dw 0x04, 0x0d, 0x09, 0x06, 0x01, 0x0e, 0x06, 0x09 + dw 0x0d, 0x02, 0x03, 0x0f, 0x00, 0x0c, 0x05, 0x0a + dw 0x07, 0x0b, 0x0e, 0x01, 0x0b, 0x07, 0x08, 0x04 + dw 0x0b, 0x06, 0x07, 0x09, 0x02, 0x08, 0x04, 0x07 + dw 0x0d, 0x0b, 0x0a, 0x00, 0x08, 0x05, 0x01, 0x0c + dw 0x00, 0x0d, 0x0c, 0x0a, 0x09, 0x02, 0x0f, 0x04 + dw 0x0e, 0x01, 0x03, 0x0f, 0x05, 0x0e, 0x06, 0x03 + ;; SBOX7 + dw 0x0b, 0x0e, 0x05, 0x00, 0x06, 0x09, 0x0a, 0x0f + dw 0x01, 0x02, 0x0c, 0x05, 0x0d, 0x07, 0x03, 0x0a + dw 0x04, 0x0d, 0x09, 0x06, 0x0f, 0x03, 0x00, 0x0c + dw 0x02, 0x08, 0x07, 0x0b, 0x08, 0x04, 0x0e, 0x01 + dw 0x08, 0x04, 0x03, 0x0f, 0x05, 0x02, 0x00, 0x0c + dw 0x0b, 0x07, 0x06, 0x09, 0x0e, 0x01, 0x09, 0x06 + dw 0x0f, 0x08, 0x0a, 0x03, 0x0c, 0x05, 0x07, 0x0a + dw 0x01, 0x0e, 0x0d, 0x00, 0x02, 0x0b, 0x04, 0x0d + +;;; Used in DOCSIS DES partial block scheduling 16 x 32bit of value 1 +align 64 +vec_ones_32b: + dd 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 + +align 64 +and_eu: + dd 0x3f003f00, 0x3f003f00, 0x3f003f00, 0x3f003f00 + dd 0x3f003f00, 0x3f003f00, 0x3f003f00, 0x3f003f00 + dd 0x3f003f00, 0x3f003f00, 0x3f003f00, 0x3f003f00 + dd 0x3f003f00, 0x3f003f00, 0x3f003f00, 0x3f003f00 + +align 64 +and_ed: + dd 0x003f003f, 0x003f003f, 0x003f003f, 0x003f003f + dd 0x003f003f, 0x003f003f, 0x003f003f, 0x003f003f + dd 0x003f003f, 0x003f003f, 0x003f003f, 0x003f003f + dd 0x003f003f, 0x003f003f, 0x003f003f, 0x003f003f + +align 64 +idx_e: + dq 0x0d0c090805040100, 0x0f0e0b0a07060302 + dq 0x1d1c191815141110, 0x1f1e1b1a17161312 + dq 0x2d2c292825242120, 0x2f2e2b2a27262322 + dq 0x3d3c393835343130, 0x3f3e3b3a37363332 + +align 64 +reg_values16bit_7: + dq 0x001f001f001f001f, 0x001f001f001f001f + dq 0x001f001f001f001f, 0x001f001f001f001f + dq 0x001f001f001f001f, 0x001f001f001f001f + dq 0x001f001f001f001f, 0x001f001f001f001f + +align 64 +shuffle_reg: + dq 0x0705060403010200, 0x0f0d0e0c0b090a08 + dq 0x1715161413111210, 0x1f1d1e1c1b191a18 + dq 0x2725262423212220, 0x2f2d2e2c2b292a28 + dq 0x3735363433313230, 0x3f3d3e3c3b393a38 + +;;; ======================================================== +;;; CODE +section .text + +;;; arg 1 : pointer to DES OOO structure +;;; arg 2 : size in bytes +align 64 +MKGLOBAL(des_x16_cbc_enc_avx512,function,internal) +des_x16_cbc_enc_avx512: + GENERIC_DES_ENC DES + ret + +;;; arg 1 : pointer to DES OOO structure +;;; arg 2 : size in bytes +align 64 +MKGLOBAL(des_x16_cbc_dec_avx512,function,internal) +des_x16_cbc_dec_avx512: + GENERIC_DES_DEC DES + ret + +;;; arg 1 : pointer to DES OOO structure +;;; arg 2 : size in bytes +align 64 +MKGLOBAL(des3_x16_cbc_enc_avx512,function,internal) +des3_x16_cbc_enc_avx512: + GENERIC_DES_ENC 3DES + ret + +;;; arg 1 : pointer to DES OOO structure +;;; arg 2 : size in bytes +align 64 +MKGLOBAL(des3_x16_cbc_dec_avx512,function,internal) +des3_x16_cbc_dec_avx512: + GENERIC_DES_DEC 3DES + ret + +;;; arg 1 : pointer to DES OOO structure +;;; arg 2 : size in bytes +align 64 +MKGLOBAL(docsis_des_x16_enc_avx512,function,internal) +docsis_des_x16_enc_avx512: + GENERIC_DES_ENC DOCSIS + ret + +;;; arg 1 : pointer to DES OOO structure +;;; arg 2 : size in bytes +align 64 +MKGLOBAL(docsis_des_x16_dec_avx512,function,internal) +docsis_des_x16_dec_avx512: + GENERIC_DES_DEC DOCSIS + ret + +%ifdef LINUX +section .note.GNU-stack noalloc noexec nowrite progbits +%endif diff --git a/src/spdk/intel-ipsec-mb/avx512/gcm128_avx512.asm b/src/spdk/intel-ipsec-mb/avx512/gcm128_avx512.asm new file mode 100644 index 000000000..f9f643b40 --- /dev/null +++ b/src/spdk/intel-ipsec-mb/avx512/gcm128_avx512.asm @@ -0,0 +1,31 @@ +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; Copyright(c) 2018, Intel Corporation All rights reserved. +; +; Redistribution and use in source and binary forms, with or without +; modification, are permitted provided that the following conditions +; are met: +; * Redistributions of source code must retain the above copyright +; notice, this list of conditions and the following disclaimer. +; * Redistributions in binary form must reproduce the above copyright +; notice, this list of conditions and the following disclaimer in +; the documentation and/or other materials provided with the +; distribution. +; * Neither the name of Intel Corporation nor the names of its +; contributors may be used to endorse or promote products derived +; from this software without specific prior written permission. +; +; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +; "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +; LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +; A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +; OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +; SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +; LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +; DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +; THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +%define GCM128_MODE 1 +%include "avx512/gcm_avx512.asm" diff --git a/src/spdk/intel-ipsec-mb/avx512/gcm128_vaes_avx512.asm b/src/spdk/intel-ipsec-mb/avx512/gcm128_vaes_avx512.asm new file mode 100644 index 000000000..2465b22dd --- /dev/null +++ b/src/spdk/intel-ipsec-mb/avx512/gcm128_vaes_avx512.asm @@ -0,0 +1,32 @@ +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; Copyright(c) 2018-2019, Intel Corporation All rights reserved. +; +; Redistribution and use in source and binary forms, with or without +; modification, are permitted provided that the following conditions +; are met: +; * Redistributions of source code must retain the above copyright +; notice, this list of conditions and the following disclaimer. +; * Redistributions in binary form must reproduce the above copyright +; notice, this list of conditions and the following disclaimer in +; the documentation and/or other materials provided with the +; distribution. +; * Neither the name of Intel Corporation nor the names of its +; contributors may be used to endorse or promote products derived +; from this software without specific prior written permission. +; +; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +; "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +; LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +; A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +; OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +; SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +; LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +; DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +; THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +%define GCM128_MODE 1 +;; single buffer implementation +%include "avx512/gcm_vaes_avx512.asm" diff --git a/src/spdk/intel-ipsec-mb/avx512/gcm192_avx512.asm b/src/spdk/intel-ipsec-mb/avx512/gcm192_avx512.asm new file mode 100644 index 000000000..403ab2f7c --- /dev/null +++ b/src/spdk/intel-ipsec-mb/avx512/gcm192_avx512.asm @@ -0,0 +1,31 @@ +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; Copyright(c) 2018, Intel Corporation All rights reserved. +; +; Redistribution and use in source and binary forms, with or without +; modification, are permitted provided that the following conditions +; are met: +; * Redistributions of source code must retain the above copyright +; notice, this list of conditions and the following disclaimer. +; * Redistributions in binary form must reproduce the above copyright +; notice, this list of conditions and the following disclaimer in +; the documentation and/or other materials provided with the +; distribution. +; * Neither the name of Intel Corporation nor the names of its +; contributors may be used to endorse or promote products derived +; from this software without specific prior written permission. +; +; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +; "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +; LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +; A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +; OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +; SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +; LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +; DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +; THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +%define GCM192_MODE 1 +%include "avx512/gcm_avx512.asm" diff --git a/src/spdk/intel-ipsec-mb/avx512/gcm192_vaes_avx512.asm b/src/spdk/intel-ipsec-mb/avx512/gcm192_vaes_avx512.asm new file mode 100644 index 000000000..348190a2a --- /dev/null +++ b/src/spdk/intel-ipsec-mb/avx512/gcm192_vaes_avx512.asm @@ -0,0 +1,32 @@ +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; Copyright(c) 2018-2019, Intel Corporation All rights reserved. +; +; Redistribution and use in source and binary forms, with or without +; modification, are permitted provided that the following conditions +; are met: +; * Redistributions of source code must retain the above copyright +; notice, this list of conditions and the following disclaimer. +; * Redistributions in binary form must reproduce the above copyright +; notice, this list of conditions and the following disclaimer in +; the documentation and/or other materials provided with the +; distribution. +; * Neither the name of Intel Corporation nor the names of its +; contributors may be used to endorse or promote products derived +; from this software without specific prior written permission. +; +; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +; "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +; LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +; A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +; OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +; SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +; LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +; DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +; THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +%define GCM192_MODE 1 +;; single buffer implementation +%include "avx512/gcm_vaes_avx512.asm" diff --git a/src/spdk/intel-ipsec-mb/avx512/gcm256_avx512.asm b/src/spdk/intel-ipsec-mb/avx512/gcm256_avx512.asm new file mode 100644 index 000000000..141b4b9ca --- /dev/null +++ b/src/spdk/intel-ipsec-mb/avx512/gcm256_avx512.asm @@ -0,0 +1,31 @@ +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; Copyright(c) 2018, Intel Corporation All rights reserved. +; +; Redistribution and use in source and binary forms, with or without +; modification, are permitted provided that the following conditions +; are met: +; * Redistributions of source code must retain the above copyright +; notice, this list of conditions and the following disclaimer. +; * Redistributions in binary form must reproduce the above copyright +; notice, this list of conditions and the following disclaimer in +; the documentation and/or other materials provided with the +; distribution. +; * Neither the name of Intel Corporation nor the names of its +; contributors may be used to endorse or promote products derived +; from this software without specific prior written permission. +; +; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +; "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +; LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +; A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +; OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +; SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +; LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +; DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +; THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +%define GCM256_MODE 1 +%include "avx512/gcm_avx512.asm" diff --git a/src/spdk/intel-ipsec-mb/avx512/gcm256_vaes_avx512.asm b/src/spdk/intel-ipsec-mb/avx512/gcm256_vaes_avx512.asm new file mode 100644 index 000000000..4daa1b361 --- /dev/null +++ b/src/spdk/intel-ipsec-mb/avx512/gcm256_vaes_avx512.asm @@ -0,0 +1,32 @@ +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; Copyright(c) 2018-2019, Intel Corporation All rights reserved. +; +; Redistribution and use in source and binary forms, with or without +; modification, are permitted provided that the following conditions +; are met: +; * Redistributions of source code must retain the above copyright +; notice, this list of conditions and the following disclaimer. +; * Redistributions in binary form must reproduce the above copyright +; notice, this list of conditions and the following disclaimer in +; the documentation and/or other materials provided with the +; distribution. +; * Neither the name of Intel Corporation nor the names of its +; contributors may be used to endorse or promote products derived +; from this software without specific prior written permission. +; +; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +; "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +; LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +; A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +; OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +; SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +; LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +; DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +; THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +%define GCM256_MODE 1 +;; single buffer implementation +%include "avx512/gcm_vaes_avx512.asm" diff --git a/src/spdk/intel-ipsec-mb/avx512/gcm_avx512.asm b/src/spdk/intel-ipsec-mb/avx512/gcm_avx512.asm new file mode 100644 index 000000000..db940ffe9 --- /dev/null +++ b/src/spdk/intel-ipsec-mb/avx512/gcm_avx512.asm @@ -0,0 +1,3536 @@ +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; Copyright(c) 2018-2019, Intel Corporation All rights reserved. +; +; Redistribution and use in source and binary forms, with or without +; modification, are permitted provided that the following conditions +; are met: +; * Redistributions of source code must retain the above copyright +; notice, this list of conditions and the following disclaimer. +; * Redistributions in binary form must reproduce the above copyright +; notice, this list of conditions and the following disclaimer in +; the documentation and/or other materials provided with the +; distribution. +; * Neither the name of Intel Corporation nor the names of its +; contributors may be used to endorse or promote products derived +; from this software without specific prior written permission. +; +; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +; "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +; LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +; A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +; OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +; SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +; LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +; DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +; THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +; +; Authors: +; Erdinc Ozturk +; Vinodh Gopal +; James Guilford +; Tomasz Kantecki +; +; +; References: +; This code was derived and highly optimized from the code described in paper: +; Vinodh Gopal et. al. Optimized Galois-Counter-Mode Implementation on Intel Architecture Processors. August, 2010 +; The details of the implementation is explained in: +; Erdinc Ozturk et. al. Enabling High-Performance Galois-Counter-Mode on Intel Architecture Processors. October, 2012. +; +; +; +; +; Assumptions: +; +; +; +; iv: +; 0 1 2 3 +; 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 +; +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +; | Salt (From the SA) | +; +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +; | Initialization Vector | +; | (This is the sequence number from IPSec header) | +; +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +; | 0x1 | +; +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +; +; +; +; AAD: +; AAD will be padded with 0 to the next 16byte multiple +; for example, assume AAD is a u32 vector +; +; if AAD is 8 bytes: +; AAD[3] = {A0, A1}; +; padded AAD in xmm register = {A1 A0 0 0} +; +; 0 1 2 3 +; 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 +; +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +; | SPI (A1) | +; +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +; | 32-bit Sequence Number (A0) | +; +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +; | 0x0 | +; +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +; +; AAD Format with 32-bit Sequence Number +; +; if AAD is 12 bytes: +; AAD[3] = {A0, A1, A2}; +; padded AAD in xmm register = {A2 A1 A0 0} +; +; 0 1 2 3 +; 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 +; +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +; | SPI (A2) | +; +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +; | 64-bit Extended Sequence Number {A1,A0} | +; | | +; +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +; | 0x0 | +; +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +; +; AAD Format with 64-bit Extended Sequence Number +; +; +; aadLen: +; Must be a multiple of 4 bytes and from the definition of the spec. +; The code additionally supports any aadLen length. +; +; TLen: +; from the definition of the spec, TLen can only be 8, 12 or 16 bytes. +; +; poly = x^128 + x^127 + x^126 + x^121 + 1 +; throughout the code, one tab and two tab indentations are used. one tab is for GHASH part, two tabs is for AES part. +; + +%include "include/os.asm" +%include "include/reg_sizes.asm" +%include "include/clear_regs.asm" +%include "include/gcm_defines.asm" +%include "include/gcm_keys_avx2_avx512.asm" + +%include "mb_mgr_datastruct.asm" +%include "job_aes_hmac.asm" +%include "include/memcpy.asm" + +%ifndef GCM128_MODE +%ifndef GCM192_MODE +%ifndef GCM256_MODE +%error "No GCM mode selected for gcm_avx512.asm!" +%endif +%endif +%endif + +;; Decide on AES-GCM key size to compile for +%ifdef GCM128_MODE +%define NROUNDS 9 +%define FN_NAME(x,y) aes_gcm_ %+ x %+ _128 %+ y %+ avx512 +%endif + +%ifdef GCM192_MODE +%define NROUNDS 11 +%define FN_NAME(x,y) aes_gcm_ %+ x %+ _192 %+ y %+ avx512 +%endif + +%ifdef GCM256_MODE +%define NROUNDS 13 +%define FN_NAME(x,y) aes_gcm_ %+ x %+ _256 %+ y %+ avx512 +%endif + +section .text +default rel + +; need to push 4 registers into stack to maintain +%define STACK_OFFSET 8*4 + +%ifidn __OUTPUT_FORMAT__, win64 + %define XMM_STORAGE 16*10 +%else + %define XMM_STORAGE 0 +%endif + +%define TMP2 16*0 ; Temporary storage for AES State 2 (State 1 is stored in an XMM register) +%define TMP3 16*1 ; Temporary storage for AES State 3 +%define TMP4 16*2 ; Temporary storage for AES State 4 +%define TMP5 16*3 ; Temporary storage for AES State 5 +%define TMP6 16*4 ; Temporary storage for AES State 6 +%define TMP7 16*5 ; Temporary storage for AES State 7 +%define TMP8 16*6 ; Temporary storage for AES State 8 +%define LOCAL_STORAGE 16*7 +%define VARIABLE_OFFSET LOCAL_STORAGE + XMM_STORAGE + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; Utility Macros +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; GHASH_MUL MACRO to implement: Data*HashKey mod (128,127,126,121,0) +; Input: A and B (128-bits each, bit-reflected) +; Output: C = A*B*x mod poly, (i.e. >>1 ) +; To compute GH = GH*HashKey mod poly, give HK = HashKey<<1 mod poly as input +; GH = GH * HK * x mod poly which is equivalent to GH*HashKey mod poly. +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +%macro GHASH_MUL 7 +%define %%GH %1 ; 16 Bytes +%define %%HK %2 ; 16 Bytes +%define %%T1 %3 +%define %%T2 %4 +%define %%T3 %5 +%define %%T4 %6 +%define %%T5 %7 + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + + vpclmulqdq %%T1, %%GH, %%HK, 0x11 ; %%T1 = a1*b1 + vpclmulqdq %%T2, %%GH, %%HK, 0x00 ; %%T2 = a0*b0 + vpclmulqdq %%T3, %%GH, %%HK, 0x01 ; %%T3 = a1*b0 + vpclmulqdq %%GH, %%GH, %%HK, 0x10 ; %%GH = a0*b1 + vpxor %%GH, %%GH, %%T3 + + + vpsrldq %%T3, %%GH, 8 ; shift-R %%GH 2 DWs + vpslldq %%GH, %%GH, 8 ; shift-L %%GH 2 DWs + + vpxor %%T1, %%T1, %%T3 + vpxor %%GH, %%GH, %%T2 + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;first phase of the reduction + vmovdqu %%T3, [rel POLY2] + + vpclmulqdq %%T2, %%T3, %%GH, 0x01 + vpslldq %%T2, %%T2, 8 ; shift-L %%T2 2 DWs + + vpxor %%GH, %%GH, %%T2 ; first phase of the reduction complete + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;second phase of the reduction + vpclmulqdq %%T2, %%T3, %%GH, 0x00 + vpsrldq %%T2, %%T2, 4 ; shift-R %%T2 1 DW (Shift-R only 1-DW to obtain 2-DWs shift-R) + + vpclmulqdq %%GH, %%T3, %%GH, 0x10 + vpslldq %%GH, %%GH, 4 ; shift-L %%GH 1 DW (Shift-L 1-DW to obtain result with no shifts) + + vpxor %%GH, %%GH, %%T2 ; second phase of the reduction complete + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + vpxor %%GH, %%GH, %%T1 ; the result is in %%GH +%endmacro + + +; In PRECOMPUTE, the commands filling Hashkey_i_k are not required for avx512 +; functions, but are kept to allow users to switch cpu architectures between calls +; of pre, init, update, and finalize. +%macro PRECOMPUTE 8 +%define %%GDATA %1 +%define %%HK %2 +%define %%T1 %3 +%define %%T2 %4 +%define %%T3 %5 +%define %%T4 %6 +%define %%T5 %7 +%define %%T6 %8 + + ; Haskey_i_k holds XORed values of the low and high parts of the Haskey_i + vmovdqa %%T5, %%HK + + GHASH_MUL %%T5, %%HK, %%T1, %%T3, %%T4, %%T6, %%T2 ; %%T5 = HashKey^2<<1 mod poly + vmovdqu [%%GDATA + HashKey_2], %%T5 ; [HashKey_2] = HashKey^2<<1 mod poly + + GHASH_MUL %%T5, %%HK, %%T1, %%T3, %%T4, %%T6, %%T2 ; %%T5 = HashKey^3<<1 mod poly + vmovdqu [%%GDATA + HashKey_3], %%T5 + + GHASH_MUL %%T5, %%HK, %%T1, %%T3, %%T4, %%T6, %%T2 ; %%T5 = HashKey^4<<1 mod poly + vmovdqu [%%GDATA + HashKey_4], %%T5 + + GHASH_MUL %%T5, %%HK, %%T1, %%T3, %%T4, %%T6, %%T2 ; %%T5 = HashKey^5<<1 mod poly + vmovdqu [%%GDATA + HashKey_5], %%T5 + + GHASH_MUL %%T5, %%HK, %%T1, %%T3, %%T4, %%T6, %%T2 ; %%T5 = HashKey^6<<1 mod poly + vmovdqu [%%GDATA + HashKey_6], %%T5 + + GHASH_MUL %%T5, %%HK, %%T1, %%T3, %%T4, %%T6, %%T2 ; %%T5 = HashKey^7<<1 mod poly + vmovdqu [%%GDATA + HashKey_7], %%T5 + + GHASH_MUL %%T5, %%HK, %%T1, %%T3, %%T4, %%T6, %%T2 ; %%T5 = HashKey^8<<1 mod poly + vmovdqu [%%GDATA + HashKey_8], %%T5 +%endmacro + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; READ_SMALL_DATA_INPUT: Packs xmm register with data when data input is less than 16 bytes. +; Returns 0 if data has length 0. +; Input: The input data (INPUT), that data's length (LENGTH). +; Output: The packed xmm register (OUTPUT). +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +%macro READ_SMALL_DATA_INPUT 4 +%define %%OUTPUT %1 ; %%OUTPUT is an xmm register +%define %%INPUT %2 +%define %%LENGTH %3 +%define %%TMP1 %4 + + lea %%TMP1, [rel byte_len_to_mask_table] +%ifidn __OUTPUT_FORMAT__, win64 + add %%TMP1, %%LENGTH + add %%TMP1, %%LENGTH + kmovw k1, [%%TMP1] +%else + kmovw k1, [%%TMP1 + %%LENGTH*2] +%endif + vmovdqu8 XWORD(%%OUTPUT){k1}{z}, [%%INPUT] + +%endmacro ; READ_SMALL_DATA_INPUT + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; CALC_AAD_HASH: Calculates the hash of the data which will not be encrypted. +; Input: The input data (A_IN), that data's length (A_LEN), and the hash key (HASH_KEY). +; Output: The hash of the data (AAD_HASH). +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +%macro CALC_AAD_HASH 13 +%define %%A_IN %1 +%define %%A_LEN %2 +%define %%AAD_HASH %3 +%define %%GDATA_KEY %4 +%define %%XTMP0 %5 ; xmm temp reg 5 +%define %%XTMP1 %6 ; xmm temp reg 5 +%define %%XTMP2 %7 +%define %%XTMP3 %8 +%define %%XTMP4 %9 +%define %%XTMP5 %10 ; xmm temp reg 5 +%define %%T1 %11 ; temp reg 1 +%define %%T2 %12 +%define %%T3 %13 + + + mov %%T1, %%A_IN ; T1 = AAD + mov %%T2, %%A_LEN ; T2 = aadLen + vpxor %%AAD_HASH, %%AAD_HASH + +%%_get_AAD_loop128: + cmp %%T2, 128 + jl %%_exit_AAD_loop128 + + vmovdqu %%XTMP0, [%%T1 + 16*0] + vpshufb %%XTMP0, [rel SHUF_MASK] + + vpxor %%XTMP0, %%AAD_HASH + + vmovdqu %%XTMP5, [%%GDATA_KEY + HashKey_8] + vpclmulqdq %%XTMP1, %%XTMP0, %%XTMP5, 0x11 ; %%T1 = a1*b1 + vpclmulqdq %%XTMP2, %%XTMP0, %%XTMP5, 0x00 ; %%T2 = a0*b0 + vpclmulqdq %%XTMP3, %%XTMP0, %%XTMP5, 0x01 ; %%T3 = a1*b0 + vpclmulqdq %%XTMP4, %%XTMP0, %%XTMP5, 0x10 ; %%T4 = a0*b1 + vpxor %%XTMP3, %%XTMP3, %%XTMP4 ; %%T3 = a1*b0 + a0*b1 + +%assign i 1 +%assign j 7 +%rep 7 + vmovdqu %%XTMP0, [%%T1 + 16*i] + vpshufb %%XTMP0, [rel SHUF_MASK] + + vmovdqu %%XTMP5, [%%GDATA_KEY + HashKey_ %+ j] + vpclmulqdq %%XTMP4, %%XTMP0, %%XTMP5, 0x11 ; %%T1 = T1 + a1*b1 + vpxor %%XTMP1, %%XTMP1, %%XTMP4 + + vpclmulqdq %%XTMP4, %%XTMP0, %%XTMP5, 0x00 ; %%T2 = T2 + a0*b0 + vpxor %%XTMP2, %%XTMP2, %%XTMP4 + + vpclmulqdq %%XTMP4, %%XTMP0, %%XTMP5, 0x01 ; %%T3 = T3 + a1*b0 + a0*b1 + vpxor %%XTMP3, %%XTMP3, %%XTMP4 + vpclmulqdq %%XTMP4, %%XTMP0, %%XTMP5, 0x10 + vpxor %%XTMP3, %%XTMP3, %%XTMP4 +%assign i (i + 1) +%assign j (j - 1) +%endrep + + vpslldq %%XTMP4, %%XTMP3, 8 ; shift-L 2 DWs + vpsrldq %%XTMP3, %%XTMP3, 8 ; shift-R 2 DWs + vpxor %%XTMP2, %%XTMP2, %%XTMP4 + vpxor %%XTMP1, %%XTMP1, %%XTMP3 ; accumulate the results in %%T1(M):%%T2(L) + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;first phase of the reduction + vmovdqa %%XTMP5, [rel POLY2] + vpclmulqdq %%XTMP0, %%XTMP5, %%XTMP2, 0x01 + vpslldq %%XTMP0, %%XTMP0, 8 ; shift-L xmm2 2 DWs + vpxor %%XTMP2, %%XTMP2, %%XTMP0 ; first phase of the reduction complete + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;second phase of the reduction + vpclmulqdq %%XTMP3, %%XTMP5, %%XTMP2, 0x00 + vpsrldq %%XTMP3, %%XTMP3, 4 ; shift-R 1 DW (Shift-R only 1-DW to obtain 2-DWs shift-R) + + vpclmulqdq %%XTMP4, %%XTMP5, %%XTMP2, 0x10 + vpslldq %%XTMP4, %%XTMP4, 4 ; shift-L 1 DW (Shift-L 1-DW to obtain result with no shifts) + + vpxor %%XTMP4, %%XTMP4, %%XTMP3 ; second phase of the reduction complete + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + vpxor %%AAD_HASH, %%XTMP1, %%XTMP4 ; the result is in %%T1 + + sub %%T2, 128 + je %%_CALC_AAD_done + + add %%T1, 128 + jmp %%_get_AAD_loop128 + +%%_exit_AAD_loop128: + cmp %%T2, 16 + jl %%_get_small_AAD_block + + ;; calculate hash_key position to start with + mov %%T3, %%T2 + and %%T3, -16 ; 1 to 7 blocks possible here + neg %%T3 + add %%T3, HashKey_1 + 16 + lea %%T3, [%%GDATA_KEY + %%T3] + + vmovdqu %%XTMP0, [%%T1] + vpshufb %%XTMP0, [rel SHUF_MASK] + + vpxor %%XTMP0, %%AAD_HASH + + vmovdqu %%XTMP5, [%%T3] + vpclmulqdq %%XTMP1, %%XTMP0, %%XTMP5, 0x11 ; %%T1 = a1*b1 + vpclmulqdq %%XTMP2, %%XTMP0, %%XTMP5, 0x00 ; %%T2 = a0*b0 + vpclmulqdq %%XTMP3, %%XTMP0, %%XTMP5, 0x01 ; %%T3 = a1*b0 + vpclmulqdq %%XTMP4, %%XTMP0, %%XTMP5, 0x10 ; %%T4 = a0*b1 + vpxor %%XTMP3, %%XTMP3, %%XTMP4 ; %%T3 = a1*b0 + a0*b1 + + add %%T3, 16 ; move to next hashkey + add %%T1, 16 ; move to next data block + sub %%T2, 16 + cmp %%T2, 16 + jl %%_AAD_reduce + +%%_AAD_blocks: + vmovdqu %%XTMP0, [%%T1] + vpshufb %%XTMP0, [rel SHUF_MASK] + + vmovdqu %%XTMP5, [%%T3] + vpclmulqdq %%XTMP4, %%XTMP0, %%XTMP5, 0x11 ; %%T1 = T1 + a1*b1 + vpxor %%XTMP1, %%XTMP1, %%XTMP4 + + vpclmulqdq %%XTMP4, %%XTMP0, %%XTMP5, 0x00 ; %%T2 = T2 + a0*b0 + vpxor %%XTMP2, %%XTMP2, %%XTMP4 + + vpclmulqdq %%XTMP4, %%XTMP0, %%XTMP5, 0x01 ; %%T3 = T3 + a1*b0 + a0*b1 + vpxor %%XTMP3, %%XTMP3, %%XTMP4 + vpclmulqdq %%XTMP4, %%XTMP0, %%XTMP5, 0x10 + vpxor %%XTMP3, %%XTMP3, %%XTMP4 + + add %%T3, 16 ; move to next hashkey + add %%T1, 16 + sub %%T2, 16 + cmp %%T2, 16 + jl %%_AAD_reduce + jmp %%_AAD_blocks + +%%_AAD_reduce: + vpslldq %%XTMP4, %%XTMP3, 8 ; shift-L 2 DWs + vpsrldq %%XTMP3, %%XTMP3, 8 ; shift-R 2 DWs + vpxor %%XTMP2, %%XTMP2, %%XTMP4 + vpxor %%XTMP1, %%XTMP1, %%XTMP3 ; accumulate the results in %%T1(M):%%T2(L) + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;first phase of the reduction + vmovdqa %%XTMP5, [rel POLY2] + vpclmulqdq %%XTMP0, %%XTMP5, %%XTMP2, 0x01 + vpslldq %%XTMP0, %%XTMP0, 8 ; shift-L xmm2 2 DWs + vpxor %%XTMP2, %%XTMP2, %%XTMP0 ; first phase of the reduction complete + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;second phase of the reduction + vpclmulqdq %%XTMP3, %%XTMP5, %%XTMP2, 0x00 + vpsrldq %%XTMP3, %%XTMP3, 4 ; shift-R 1 DW (Shift-R only 1-DW to obtain 2-DWs shift-R) + + vpclmulqdq %%XTMP4, %%XTMP5, %%XTMP2, 0x10 + vpslldq %%XTMP4, %%XTMP4, 4 ; shift-L 1 DW (Shift-L 1-DW to obtain result with no shifts) + + vpxor %%XTMP4, %%XTMP4, %%XTMP3 ; second phase of the reduction complete + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + vpxor %%AAD_HASH, %%XTMP1, %%XTMP4 ; the result is in %%T1 + + or %%T2, %%T2 + je %%_CALC_AAD_done + +%%_get_small_AAD_block: + vmovdqu %%XTMP0, [%%GDATA_KEY + HashKey] + READ_SMALL_DATA_INPUT %%XTMP1, %%T1, %%T2, %%T3 + ;byte-reflect the AAD data + vpshufb %%XTMP1, [rel SHUF_MASK] + vpxor %%AAD_HASH, %%XTMP1 + GHASH_MUL %%AAD_HASH, %%XTMP0, %%XTMP1, %%XTMP2, %%XTMP3, %%XTMP4, %%XTMP5 + +%%_CALC_AAD_done: + +%endmacro ; CALC_AAD_HASH + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; PARTIAL_BLOCK: Handles encryption/decryption and the tag partial blocks between update calls. +; Requires the input data be at least 1 byte long. +; Input: gcm_key_data * (GDATA_KEY), gcm_context_data *(GDATA_CTX), input text (PLAIN_CYPH_IN), +; input text length (PLAIN_CYPH_LEN), the current data offset (DATA_OFFSET), +; and whether encoding or decoding (ENC_DEC) +; Output: A cypher of the first partial block (CYPH_PLAIN_OUT), and updated GDATA_CTX +; Clobbers rax, r10, r12, r13, r15, xmm0, xmm1, xmm2, xmm3, xmm5, xmm6, xmm9, xmm10, xmm11, xmm13 +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +%macro PARTIAL_BLOCK 8 +%define %%GDATA_KEY %1 +%define %%GDATA_CTX %2 +%define %%CYPH_PLAIN_OUT %3 +%define %%PLAIN_CYPH_IN %4 +%define %%PLAIN_CYPH_LEN %5 +%define %%DATA_OFFSET %6 +%define %%AAD_HASH %7 +%define %%ENC_DEC %8 + + mov r13, [%%GDATA_CTX + PBlockLen] + cmp r13, 0 + je %%_partial_block_done ;Leave Macro if no partial blocks + + cmp %%PLAIN_CYPH_LEN, 16 ;Read in input data without over reading + jl %%_fewer_than_16_bytes + VXLDR xmm1, [%%PLAIN_CYPH_IN] ;If more than 16 bytes of data, just fill the xmm register + jmp %%_data_read + +%%_fewer_than_16_bytes: + lea r10, [%%PLAIN_CYPH_IN] + READ_SMALL_DATA_INPUT xmm1, r10, %%PLAIN_CYPH_LEN, rax + +%%_data_read: ;Finished reading in data + + vmovdqu xmm9, [%%GDATA_CTX + PBlockEncKey] ;xmm9 = my_ctx_data.partial_block_enc_key + vmovdqu xmm13, [%%GDATA_KEY + HashKey] + + lea r12, [rel SHIFT_MASK] + + add r12, r13 ; adjust the shuffle mask pointer to be able to shift r13 bytes (16-r13 is the number of bytes in plaintext mod 16) + vmovdqu xmm2, [r12] ; get the appropriate shuffle mask + vpshufb xmm9, xmm2 ;shift right r13 bytes + +%ifidn %%ENC_DEC, DEC + vmovdqa xmm3, xmm1 +%endif + vpxor xmm9, xmm1 ; Cyphertext XOR E(K, Yn) + + mov r15, %%PLAIN_CYPH_LEN + add r15, r13 + sub r15, 16 ;Set r15 to be the amount of data left in CYPH_PLAIN_IN after filling the block + jge %%_no_extra_mask ;Determine if if partial block is not being filled and shift mask accordingly + sub r12, r15 +%%_no_extra_mask: + + vmovdqu xmm1, [r12 + ALL_F - SHIFT_MASK]; get the appropriate mask to mask out bottom r13 bytes of xmm9 + vpand xmm9, xmm1 ; mask out bottom r13 bytes of xmm9 + +%ifidn %%ENC_DEC, DEC + vpand xmm3, xmm1 + vpshufb xmm3, [rel SHUF_MASK] + vpshufb xmm3, xmm2 + vpxor %%AAD_HASH, xmm3 +%else + vpshufb xmm9, [rel SHUF_MASK] + vpshufb xmm9, xmm2 + vpxor %%AAD_HASH, xmm9 +%endif + cmp r15,0 + jl %%_partial_incomplete + + GHASH_MUL %%AAD_HASH, xmm13, xmm0, xmm10, xmm11, xmm5, xmm6 ;GHASH computation for the last <16 Byte block + xor rax,rax + mov [%%GDATA_CTX + PBlockLen], rax + jmp %%_enc_dec_done +%%_partial_incomplete: +%ifidn __OUTPUT_FORMAT__, win64 + mov rax, %%PLAIN_CYPH_LEN + add [%%GDATA_CTX + PBlockLen], rax +%else + add [%%GDATA_CTX + PBlockLen], %%PLAIN_CYPH_LEN +%endif +%%_enc_dec_done: + vmovdqu [%%GDATA_CTX + AadHash], %%AAD_HASH + +%ifidn %%ENC_DEC, ENC + vpshufb xmm9, [rel SHUF_MASK] ; shuffle xmm9 back to output as ciphertext + vpshufb xmm9, xmm2 +%endif + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ; output encrypted Bytes + cmp r15,0 + jl %%_partial_fill + mov r12, r13 + mov r13, 16 + sub r13, r12 ; Set r13 to be the number of bytes to write out + jmp %%_count_set +%%_partial_fill: + mov r13, %%PLAIN_CYPH_LEN +%%_count_set: + lea rax, [rel byte_len_to_mask_table] + kmovw k1, [rax + r13*2] + vmovdqu8 [%%CYPH_PLAIN_OUT + %%DATA_OFFSET]{k1}, xmm9 + add %%DATA_OFFSET, r13 +%%_partial_block_done: +%endmacro ; PARTIAL_BLOCK + + +%macro GHASH_SINGLE_MUL 9 +%define %%GDATA %1 +%define %%HASHKEY %2 +%define %%CIPHER %3 +%define %%STATE_11 %4 +%define %%STATE_00 %5 +%define %%STATE_MID %6 +%define %%T1 %7 +%define %%T2 %8 +%define %%FIRST %9 + + vmovdqu %%T1, [%%GDATA + %%HASHKEY] +%ifidn %%FIRST, first + vpclmulqdq %%STATE_11, %%CIPHER, %%T1, 0x11 ; %%T4 = a1*b1 + vpclmulqdq %%STATE_00, %%CIPHER, %%T1, 0x00 ; %%T4_2 = a0*b0 + vpclmulqdq %%STATE_MID, %%CIPHER, %%T1, 0x01 ; %%T6 = a1*b0 + vpclmulqdq %%T2, %%CIPHER, %%T1, 0x10 ; %%T5 = a0*b1 + vpxor %%STATE_MID, %%STATE_MID, %%T2 +%else + vpclmulqdq %%T2, %%CIPHER, %%T1, 0x11 + vpxor %%STATE_11, %%STATE_11, %%T2 + + vpclmulqdq %%T2, %%CIPHER, %%T1, 0x00 + vpxor %%STATE_00, %%STATE_00, %%T2 + + vpclmulqdq %%T2, %%CIPHER, %%T1, 0x01 + vpxor %%STATE_MID, %%STATE_MID, %%T2 + + vpclmulqdq %%T2, %%CIPHER, %%T1, 0x10 + vpxor %%STATE_MID, %%STATE_MID, %%T2 +%endif + +%endmacro + +; if a = number of total plaintext bytes +; b = floor(a/16) +; %%num_initial_blocks = b mod 8; +; encrypt the initial %%num_initial_blocks blocks and apply ghash on the ciphertext +; %%GDATA_KEY, %%CYPH_PLAIN_OUT, %%PLAIN_CYPH_IN, r14 are used as a pointer only, not modified. +; Updated AAD_HASH is returned in %%T3 + +%macro INITIAL_BLOCKS 23 +%define %%GDATA_KEY %1 +%define %%CYPH_PLAIN_OUT %2 +%define %%PLAIN_CYPH_IN %3 +%define %%LENGTH %4 +%define %%DATA_OFFSET %5 +%define %%num_initial_blocks %6 ; can be 0, 1, 2, 3, 4, 5, 6 or 7 +%define %%T1 %7 +%define %%T2 %8 +%define %%T3 %9 +%define %%T4 %10 +%define %%T5 %11 +%define %%CTR %12 +%define %%XMM1 %13 +%define %%XMM2 %14 +%define %%XMM3 %15 +%define %%XMM4 %16 +%define %%XMM5 %17 +%define %%XMM6 %18 +%define %%XMM7 %19 +%define %%XMM8 %20 +%define %%T6 %21 +%define %%T_key %22 +%define %%ENC_DEC %23 + +%assign i (8-%%num_initial_blocks) + ;; Move AAD_HASH to temp reg + vmovdqu %%T2, %%XMM8 + ;; Start AES for %%num_initial_blocks blocks + ;; vmovdqu %%CTR, [%%GDATA_CTX + CurCount] ; %%CTR = Y0 + +%assign i (9-%%num_initial_blocks) +%rep %%num_initial_blocks + vpaddd %%CTR, %%CTR, [rel ONE] ; INCR Y0 + vmovdqa reg(i), %%CTR + vpshufb reg(i), [rel SHUF_MASK] ; perform a 16Byte swap +%assign i (i+1) +%endrep + +%if(%%num_initial_blocks>0) +vmovdqu %%T_key, [%%GDATA_KEY+16*0] +%assign i (9-%%num_initial_blocks) +%rep %%num_initial_blocks + vpxor reg(i),reg(i),%%T_key +%assign i (i+1) +%endrep + +%assign j 1 +%rep NROUNDS +vmovdqu %%T_key, [%%GDATA_KEY+16*j] +%assign i (9-%%num_initial_blocks) +%rep %%num_initial_blocks + vaesenc reg(i),%%T_key +%assign i (i+1) +%endrep + +%assign j (j+1) +%endrep + + +vmovdqu %%T_key, [%%GDATA_KEY+16*j] +%assign i (9-%%num_initial_blocks) +%rep %%num_initial_blocks + vaesenclast reg(i),%%T_key +%assign i (i+1) +%endrep + +%endif ; %if(%%num_initial_blocks>0) + + + +%assign i (9-%%num_initial_blocks) +%rep %%num_initial_blocks + VXLDR %%T1, [%%PLAIN_CYPH_IN + %%DATA_OFFSET] + vpxor reg(i), reg(i), %%T1 + ;; Write back ciphertext for %%num_initial_blocks blocks + VXSTR [%%CYPH_PLAIN_OUT + %%DATA_OFFSET], reg(i) + add %%DATA_OFFSET, 16 + %ifidn %%ENC_DEC, DEC + vmovdqa reg(i), %%T1 + %endif + ;; Prepare ciphertext for GHASH computations + vpshufb reg(i), [rel SHUF_MASK] +%assign i (i+1) +%endrep + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +%assign i (9-%%num_initial_blocks) +%if(%%num_initial_blocks>0) + vmovdqa %%T3, reg(i) +%assign i (i+1) +%endif +%if %%num_initial_blocks>1 +%rep %%num_initial_blocks-1 + vmovdqu [rsp + TMP %+ i], reg(i) +%assign i (i+1) +%endrep +%endif + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; Prepare 8 counter blocks and perform rounds of AES cipher on + ;; them, load plain/cipher text and store cipher/plain text. + ;; Stitch GHASH computation in between AES rounds. + vpaddd %%XMM1, %%CTR, [rel ONE] ; INCR Y0 + vpaddd %%XMM2, %%CTR, [rel TWO] ; INCR Y0 + vpaddd %%XMM3, %%XMM1, [rel TWO] ; INCR Y0 + vpaddd %%XMM4, %%XMM2, [rel TWO] ; INCR Y0 + vpaddd %%XMM5, %%XMM3, [rel TWO] ; INCR Y0 + vpaddd %%XMM6, %%XMM4, [rel TWO] ; INCR Y0 + vpaddd %%XMM7, %%XMM5, [rel TWO] ; INCR Y0 + vpaddd %%XMM8, %%XMM6, [rel TWO] ; INCR Y0 + vmovdqa %%CTR, %%XMM8 + + vpshufb %%XMM1, [rel SHUF_MASK] ; perform a 16Byte swap + vpshufb %%XMM2, [rel SHUF_MASK] ; perform a 16Byte swap + vpshufb %%XMM3, [rel SHUF_MASK] ; perform a 16Byte swap + vpshufb %%XMM4, [rel SHUF_MASK] ; perform a 16Byte swap + vpshufb %%XMM5, [rel SHUF_MASK] ; perform a 16Byte swap + vpshufb %%XMM6, [rel SHUF_MASK] ; perform a 16Byte swap + vpshufb %%XMM7, [rel SHUF_MASK] ; perform a 16Byte swap + vpshufb %%XMM8, [rel SHUF_MASK] ; perform a 16Byte swap + + vmovdqu %%T_key, [%%GDATA_KEY+16*0] + vpxor %%XMM1, %%XMM1, %%T_key + vpxor %%XMM2, %%XMM2, %%T_key + vpxor %%XMM3, %%XMM3, %%T_key + vpxor %%XMM4, %%XMM4, %%T_key + vpxor %%XMM5, %%XMM5, %%T_key + vpxor %%XMM6, %%XMM6, %%T_key + vpxor %%XMM7, %%XMM7, %%T_key + vpxor %%XMM8, %%XMM8, %%T_key + +%assign i (8-%%num_initial_blocks) +%assign j (9-%%num_initial_blocks) +%assign k (%%num_initial_blocks) + +%define %%T4_2 %%T4 +%if(%%num_initial_blocks>0) + ;; Hash in AES state + ;; T2 - incoming AAD hash + vpxor %%T2, %%T3 + + ;; GDATA, HASHKEY, CIPHER, + ;; STATE_11, STATE_00, STATE_MID, T1, T2 + GHASH_SINGLE_MUL %%GDATA_KEY, HashKey_ %+ k, %%T2, \ + %%T1, %%T4, %%T6, %%T5, %%T3, first +%endif + + vmovdqu %%T_key, [%%GDATA_KEY+16*1] + vaesenc %%XMM1, %%T_key + vaesenc %%XMM2, %%T_key + vaesenc %%XMM3, %%T_key + vaesenc %%XMM4, %%T_key + vaesenc %%XMM5, %%T_key + vaesenc %%XMM6, %%T_key + vaesenc %%XMM7, %%T_key + vaesenc %%XMM8, %%T_key + + vmovdqu %%T_key, [%%GDATA_KEY+16*2] + vaesenc %%XMM1, %%T_key + vaesenc %%XMM2, %%T_key + vaesenc %%XMM3, %%T_key + vaesenc %%XMM4, %%T_key + vaesenc %%XMM5, %%T_key + vaesenc %%XMM6, %%T_key + vaesenc %%XMM7, %%T_key + vaesenc %%XMM8, %%T_key + +%assign i (i+1) +%assign j (j+1) +%assign k (k-1) +%if(%%num_initial_blocks>1) + ;; GDATA, HASHKEY, CIPHER, + ;; STATE_11, STATE_00, STATE_MID, T1, T2 + vmovdqu %%T2, [rsp + TMP %+ j] + GHASH_SINGLE_MUL %%GDATA_KEY, HashKey_ %+ k, %%T2, \ + %%T1, %%T4, %%T6, %%T5, %%T3, not_first +%endif + + vmovdqu %%T_key, [%%GDATA_KEY+16*3] + vaesenc %%XMM1, %%T_key + vaesenc %%XMM2, %%T_key + vaesenc %%XMM3, %%T_key + vaesenc %%XMM4, %%T_key + vaesenc %%XMM5, %%T_key + vaesenc %%XMM6, %%T_key + vaesenc %%XMM7, %%T_key + vaesenc %%XMM8, %%T_key + + vmovdqu %%T_key, [%%GDATA_KEY+16*4] + vaesenc %%XMM1, %%T_key + vaesenc %%XMM2, %%T_key + vaesenc %%XMM3, %%T_key + vaesenc %%XMM4, %%T_key + vaesenc %%XMM5, %%T_key + vaesenc %%XMM6, %%T_key + vaesenc %%XMM7, %%T_key + vaesenc %%XMM8, %%T_key + +%assign i (i+1) +%assign j (j+1) +%assign k (k-1) +%if(%%num_initial_blocks>2) + ;; GDATA, HASHKEY, CIPHER, + ;; STATE_11, STATE_00, STATE_MID, T1, T2 + vmovdqu %%T2, [rsp + TMP %+ j] + GHASH_SINGLE_MUL %%GDATA_KEY, HashKey_ %+ k, %%T2, \ + %%T1, %%T4, %%T6, %%T5, %%T3, not_first +%endif + +%assign i (i+1) +%assign j (j+1) +%assign k (k-1) +%if(%%num_initial_blocks>3) + ;; GDATA, HASHKEY, CIPHER, + ;; STATE_11, STATE_00, STATE_MID, T1, T2 + vmovdqu %%T2, [rsp + TMP %+ j] + GHASH_SINGLE_MUL %%GDATA_KEY, HashKey_ %+ k, %%T2, \ + %%T1, %%T4, %%T6, %%T5, %%T3, not_first +%endif + + vmovdqu %%T_key, [%%GDATA_KEY+16*5] + vaesenc %%XMM1, %%T_key + vaesenc %%XMM2, %%T_key + vaesenc %%XMM3, %%T_key + vaesenc %%XMM4, %%T_key + vaesenc %%XMM5, %%T_key + vaesenc %%XMM6, %%T_key + vaesenc %%XMM7, %%T_key + vaesenc %%XMM8, %%T_key + + vmovdqu %%T_key, [%%GDATA_KEY+16*6] + vaesenc %%XMM1, %%T_key + vaesenc %%XMM2, %%T_key + vaesenc %%XMM3, %%T_key + vaesenc %%XMM4, %%T_key + vaesenc %%XMM5, %%T_key + vaesenc %%XMM6, %%T_key + vaesenc %%XMM7, %%T_key + vaesenc %%XMM8, %%T_key + +%assign i (i+1) +%assign j (j+1) +%assign k (k-1) +%if(%%num_initial_blocks>4) + ;; GDATA, HASHKEY, CIPHER, + ;; STATE_11, STATE_00, STATE_MID, T1, T2 + vmovdqu %%T2, [rsp + TMP %+ j] + GHASH_SINGLE_MUL %%GDATA_KEY, HashKey_ %+ k, %%T2, \ + %%T1, %%T4, %%T6, %%T5, %%T3, not_first +%endif + + vmovdqu %%T_key, [%%GDATA_KEY+16*7] + vaesenc %%XMM1, %%T_key + vaesenc %%XMM2, %%T_key + vaesenc %%XMM3, %%T_key + vaesenc %%XMM4, %%T_key + vaesenc %%XMM5, %%T_key + vaesenc %%XMM6, %%T_key + vaesenc %%XMM7, %%T_key + vaesenc %%XMM8, %%T_key + + vmovdqu %%T_key, [%%GDATA_KEY+16*8] + vaesenc %%XMM1, %%T_key + vaesenc %%XMM2, %%T_key + vaesenc %%XMM3, %%T_key + vaesenc %%XMM4, %%T_key + vaesenc %%XMM5, %%T_key + vaesenc %%XMM6, %%T_key + vaesenc %%XMM7, %%T_key + vaesenc %%XMM8, %%T_key + +%assign i (i+1) +%assign j (j+1) +%assign k (k-1) +%if(%%num_initial_blocks>5) + ;; GDATA, HASHKEY, CIPHER, + ;; STATE_11, STATE_00, STATE_MID, T1, T2 + vmovdqu %%T2, [rsp + TMP %+ j] + GHASH_SINGLE_MUL %%GDATA_KEY, HashKey_ %+ k, %%T2, \ + %%T1, %%T4, %%T6, %%T5, %%T3, not_first +%endif + + vmovdqu %%T_key, [%%GDATA_KEY+16*9] + vaesenc %%XMM1, %%T_key + vaesenc %%XMM2, %%T_key + vaesenc %%XMM3, %%T_key + vaesenc %%XMM4, %%T_key + vaesenc %%XMM5, %%T_key + vaesenc %%XMM6, %%T_key + vaesenc %%XMM7, %%T_key + vaesenc %%XMM8, %%T_key + +%ifndef GCM128_MODE + vmovdqu %%T_key, [%%GDATA_KEY+16*10] + vaesenc %%XMM1, %%T_key + vaesenc %%XMM2, %%T_key + vaesenc %%XMM3, %%T_key + vaesenc %%XMM4, %%T_key + vaesenc %%XMM5, %%T_key + vaesenc %%XMM6, %%T_key + vaesenc %%XMM7, %%T_key + vaesenc %%XMM8, %%T_key +%endif + +%assign i (i+1) +%assign j (j+1) +%assign k (k-1) +%if(%%num_initial_blocks>6) + ;; GDATA, HASHKEY, CIPHER, + ;; STATE_11, STATE_00, STATE_MID, T1, T2 + vmovdqu %%T2, [rsp + TMP %+ j] + GHASH_SINGLE_MUL %%GDATA_KEY, HashKey_ %+ k, %%T2, \ + %%T1, %%T4, %%T6, %%T5, %%T3, not_first +%endif + +%ifdef GCM128_MODE + vmovdqu %%T_key, [%%GDATA_KEY+16*10] + vaesenclast %%XMM1, %%T_key + vaesenclast %%XMM2, %%T_key + vaesenclast %%XMM3, %%T_key + vaesenclast %%XMM4, %%T_key + vaesenclast %%XMM5, %%T_key + vaesenclast %%XMM6, %%T_key + vaesenclast %%XMM7, %%T_key + vaesenclast %%XMM8, %%T_key +%endif + +%ifdef GCM192_MODE + vmovdqu %%T_key, [%%GDATA_KEY+16*11] + vaesenc %%XMM1, %%T_key + vaesenc %%XMM2, %%T_key + vaesenc %%XMM3, %%T_key + vaesenc %%XMM4, %%T_key + vaesenc %%XMM5, %%T_key + vaesenc %%XMM6, %%T_key + vaesenc %%XMM7, %%T_key + vaesenc %%XMM8, %%T_key + + vmovdqu %%T_key, [%%GDATA_KEY+16*12] + vaesenclast %%XMM1, %%T_key + vaesenclast %%XMM2, %%T_key + vaesenclast %%XMM3, %%T_key + vaesenclast %%XMM4, %%T_key + vaesenclast %%XMM5, %%T_key + vaesenclast %%XMM6, %%T_key + vaesenclast %%XMM7, %%T_key + vaesenclast %%XMM8, %%T_key +%endif +%ifdef GCM256_MODE + vmovdqu %%T_key, [%%GDATA_KEY+16*11] + vaesenc %%XMM1, %%T_key + vaesenc %%XMM2, %%T_key + vaesenc %%XMM3, %%T_key + vaesenc %%XMM4, %%T_key + vaesenc %%XMM5, %%T_key + vaesenc %%XMM6, %%T_key + vaesenc %%XMM7, %%T_key + vaesenc %%XMM8, %%T_key + + vmovdqu %%T_key, [%%GDATA_KEY+16*12] + vaesenc %%XMM1, %%T_key + vaesenc %%XMM2, %%T_key + vaesenc %%XMM3, %%T_key + vaesenc %%XMM4, %%T_key + vaesenc %%XMM5, %%T_key + vaesenc %%XMM6, %%T_key + vaesenc %%XMM7, %%T_key + vaesenc %%XMM8, %%T_key +%endif + +%assign i (i+1) +%assign j (j+1) +%assign k (k-1) +%if(%%num_initial_blocks>7) + ;; GDATA, HASHKEY, CIPHER, + ;; STATE_11, STATE_00, STATE_MID, T1, T2 + vmovdqu %%T2, [rsp + TMP %+ j] + GHASH_SINGLE_MUL %%GDATA_KEY, HashKey_ %+ k, %%T2, \ + %%T1, %%T4, %%T6, %%T5, %%T3, not_first +%endif + +%ifdef GCM256_MODE ; GCM256 + vmovdqu %%T_key, [%%GDATA_KEY+16*13] + vaesenc %%XMM1, %%T_key + vaesenc %%XMM2, %%T_key + vaesenc %%XMM3, %%T_key + vaesenc %%XMM4, %%T_key + vaesenc %%XMM5, %%T_key + vaesenc %%XMM6, %%T_key + vaesenc %%XMM7, %%T_key + vaesenc %%XMM8, %%T_key + + vmovdqu %%T_key, [%%GDATA_KEY+16*14] + vaesenclast %%XMM1, %%T_key + vaesenclast %%XMM2, %%T_key + vaesenclast %%XMM3, %%T_key + vaesenclast %%XMM4, %%T_key + vaesenclast %%XMM5, %%T_key + vaesenclast %%XMM6, %%T_key + vaesenclast %%XMM7, %%T_key + vaesenclast %%XMM8, %%T_key +%endif ; GCM256 mode + +%if(%%num_initial_blocks>0) + vpsrldq %%T3, %%T6, 8 ; shift-R %%T2 2 DWs + vpslldq %%T6, %%T6, 8 ; shift-L %%T3 2 DWs + vpxor %%T1, %%T1, %%T3 ; accumulate the results in %%T1:%%T4 + vpxor %%T4, %%T6, %%T4 + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ; First phase of the reduction + vmovdqu %%T3, [rel POLY2] + + vpclmulqdq %%T2, %%T3, %%T4, 0x01 + vpslldq %%T2, %%T2, 8 ; shift-L xmm2 2 DWs + + ;; First phase of the reduction complete + vpxor %%T4, %%T4, %%T2 + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ; Second phase of the reduction + vpclmulqdq %%T2, %%T3, %%T4, 0x00 + ;; Shift-R xmm2 1 DW (Shift-R only 1-DW to obtain 2-DWs shift-R) + vpsrldq %%T2, %%T2, 4 + + vpclmulqdq %%T4, %%T3, %%T4, 0x10 + ;; Shift-L xmm0 1 DW (Shift-L 1-DW to obtain result with no shifts) + vpslldq %%T4, %%T4, 4 + ;; Second phase of the reduction complete + vpxor %%T4, %%T4, %%T2 + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ; The result is in %%T3 + vpxor %%T3, %%T1, %%T4 +%else + ;; The hash should end up in T3 + vmovdqa %%T3, %%T2 +%endif + + ;; Final hash is now in T3 +%if %%num_initial_blocks > 0 + ;; NOTE: obsolete in case %%num_initial_blocks = 0 + sub %%LENGTH, 16*%%num_initial_blocks +%endif + + VXLDR %%T1, [%%PLAIN_CYPH_IN + %%DATA_OFFSET + 16*0] + vpxor %%XMM1, %%XMM1, %%T1 + VXSTR [%%CYPH_PLAIN_OUT + %%DATA_OFFSET + 16*0], %%XMM1 + %ifidn %%ENC_DEC, DEC + vmovdqa %%XMM1, %%T1 + %endif + + VXLDR %%T1, [%%PLAIN_CYPH_IN + %%DATA_OFFSET + 16*1] + vpxor %%XMM2, %%XMM2, %%T1 + VXSTR [%%CYPH_PLAIN_OUT + %%DATA_OFFSET + 16*1], %%XMM2 + %ifidn %%ENC_DEC, DEC + vmovdqa %%XMM2, %%T1 + %endif + + VXLDR %%T1, [%%PLAIN_CYPH_IN + %%DATA_OFFSET + 16*2] + vpxor %%XMM3, %%XMM3, %%T1 + VXSTR [%%CYPH_PLAIN_OUT + %%DATA_OFFSET + 16*2], %%XMM3 + %ifidn %%ENC_DEC, DEC + vmovdqa %%XMM3, %%T1 + %endif + + VXLDR %%T1, [%%PLAIN_CYPH_IN + %%DATA_OFFSET + 16*3] + vpxor %%XMM4, %%XMM4, %%T1 + VXSTR [%%CYPH_PLAIN_OUT + %%DATA_OFFSET + 16*3], %%XMM4 + %ifidn %%ENC_DEC, DEC + vmovdqa %%XMM4, %%T1 + %endif + + VXLDR %%T1, [%%PLAIN_CYPH_IN + %%DATA_OFFSET + 16*4] + vpxor %%XMM5, %%XMM5, %%T1 + VXSTR [%%CYPH_PLAIN_OUT + %%DATA_OFFSET + 16*4], %%XMM5 + %ifidn %%ENC_DEC, DEC + vmovdqa %%XMM5, %%T1 + %endif + + VXLDR %%T1, [%%PLAIN_CYPH_IN + %%DATA_OFFSET + 16*5] + vpxor %%XMM6, %%XMM6, %%T1 + VXSTR [%%CYPH_PLAIN_OUT + %%DATA_OFFSET + 16*5], %%XMM6 + %ifidn %%ENC_DEC, DEC + vmovdqa %%XMM6, %%T1 + %endif + + VXLDR %%T1, [%%PLAIN_CYPH_IN + %%DATA_OFFSET + 16*6] + vpxor %%XMM7, %%XMM7, %%T1 + VXSTR [%%CYPH_PLAIN_OUT + %%DATA_OFFSET + 16*6], %%XMM7 + %ifidn %%ENC_DEC, DEC + vmovdqa %%XMM7, %%T1 + %endif + +%if %%num_initial_blocks > 0 + ;; NOTE: 'jl' is never taken for %%num_initial_blocks = 0 + ;; This macro is executed for lenght 128 and up, + ;; zero length is checked in GCM_ENC_DEC. + ;; If the last block is partial then the xor will be done later + ;; in ENCRYPT_FINAL_PARTIAL_BLOCK. + ;; We know it's partial if LENGTH - 16*num_initial_blocks < 128 + cmp %%LENGTH, 128 + jl %%_initial_skip_last_word_write +%endif + VXLDR %%T1, [%%PLAIN_CYPH_IN + %%DATA_OFFSET + 16*7] + vpxor %%XMM8, %%XMM8, %%T1 + VXSTR [%%CYPH_PLAIN_OUT + %%DATA_OFFSET + 16*7], %%XMM8 + %ifidn %%ENC_DEC, DEC + vmovdqa %%XMM8, %%T1 + %endif + + ;; Update %%LENGTH with the number of blocks processed + sub %%LENGTH, 16 + add %%DATA_OFFSET, 16 +%%_initial_skip_last_word_write: + sub %%LENGTH, 128-16 + add %%DATA_OFFSET, 128-16 + + vpshufb %%XMM1, [rel SHUF_MASK] ; perform a 16Byte swap + ;; Combine GHASHed value with the corresponding ciphertext + vpxor %%XMM1, %%XMM1, %%T3 + vpshufb %%XMM2, [rel SHUF_MASK] ; perform a 16Byte swap + vpshufb %%XMM3, [rel SHUF_MASK] ; perform a 16Byte swap + vpshufb %%XMM4, [rel SHUF_MASK] ; perform a 16Byte swap + vpshufb %%XMM5, [rel SHUF_MASK] ; perform a 16Byte swap + vpshufb %%XMM6, [rel SHUF_MASK] ; perform a 16Byte swap + vpshufb %%XMM7, [rel SHUF_MASK] ; perform a 16Byte swap + vpshufb %%XMM8, [rel SHUF_MASK] ; perform a 16Byte swap + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +%%_initial_blocks_done: + + +%endmacro + +;;; INITIAL_BLOCKS macro with support for a partial final block. +;;; num_initial_blocks is expected to include the partial final block +;;; in the count. +%macro INITIAL_BLOCKS_PARTIAL 25 +%define %%GDATA_KEY %1 +%define %%GDATA_CTX %2 +%define %%CYPH_PLAIN_OUT %3 +%define %%PLAIN_CYPH_IN %4 +%define %%LENGTH %5 +%define %%DATA_OFFSET %6 +%define %%num_initial_blocks %7 ; can be 1, 2, 3, 4, 5, 6 or 7 (not 0) +%define %%T1 %8 +%define %%T2 %9 +%define %%T3 %10 ; [out] hash value +%define %%T4 %11 +%define %%T5 %12 +%define %%CTR %13 +%define %%XMM1 %14 +%define %%XMM2 %15 +%define %%XMM3 %16 +%define %%XMM4 %17 +%define %%XMM5 %18 +%define %%XMM6 %19 +%define %%XMM7 %20 +%define %%XMM8 %21 ; [in] hash value +%define %%T6 %22 +%define %%T_key %23 +%define %%ENC_DEC %24 +%define %%INSTANCE_TYPE %25 + + ;; Move AAD_HASH to temp reg + vmovdqu %%T2, %%XMM8 + +%assign i (9-%%num_initial_blocks) +%rep %%num_initial_blocks + ;; Compute AES counters + vpaddd %%CTR, %%CTR, [rel ONE] ; INCR Y0 + vmovdqa reg(i), %%CTR + vpshufb reg(i), [rel SHUF_MASK] ; perform a 16Byte swap +%assign i (i+1) +%endrep + +vmovdqu %%T_key, [%%GDATA_KEY+16*0] +%assign i (9-%%num_initial_blocks) +%rep %%num_initial_blocks + ; Start AES for %%num_initial_blocks blocks + vpxor reg(i),reg(i),%%T_key +%assign i (i+1) +%endrep + +%assign j 1 +%rep NROUNDS +vmovdqu %%T_key, [%%GDATA_KEY+16*j] +%assign i (9-%%num_initial_blocks) +%rep %%num_initial_blocks + vaesenc reg(i),%%T_key +%assign i (i+1) +%endrep + +%assign j (j+1) +%endrep + + +vmovdqu %%T_key, [%%GDATA_KEY+16*j] +%assign i (9-%%num_initial_blocks) +%rep %%num_initial_blocks + vaesenclast reg(i),%%T_key +%assign i (i+1) +%endrep + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;; Hash all but the last block of data +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +%assign i (9-%%num_initial_blocks) +%rep %%num_initial_blocks-1 + ;; Encrypt the message for all but the last block + VXLDR %%T1, [%%PLAIN_CYPH_IN + %%DATA_OFFSET] + vpxor reg(i), reg(i), %%T1 + ;; write back ciphertext for %%num_initial_blocks blocks + VXSTR [%%CYPH_PLAIN_OUT + %%DATA_OFFSET], reg(i) + add %%DATA_OFFSET, 16 +%ifidn %%ENC_DEC, DEC + vmovdqa reg(i), %%T1 +%endif + ;; Prepare ciphertext for GHASH computations + vpshufb reg(i), [rel SHUF_MASK] +%assign i (i+1) +%endrep + +%if %%num_initial_blocks > 1 + ;; The final block of data may be <16B + sub %%LENGTH, 16*(%%num_initial_blocks-1) +%endif + +%if %%num_initial_blocks < 8 + ;; NOTE: the 'jl' is always taken for num_initial_blocks = 8. + ;; This is run in the context of GCM_ENC_DEC_SMALL for length < 128. + cmp %%LENGTH, 16 + jl %%_small_initial_partial_block + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;; Handle a full length final block - encrypt and hash all blocks +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + + sub %%LENGTH, 16 + mov [%%GDATA_CTX + PBlockLen], %%LENGTH + + ;; Encrypt the message + VXLDR %%T1, [%%PLAIN_CYPH_IN + %%DATA_OFFSET] + vpxor reg(i), reg(i), %%T1 + ;; write back ciphertext for %%num_initial_blocks blocks + VXSTR [%%CYPH_PLAIN_OUT + %%DATA_OFFSET], reg(i) + add %%DATA_OFFSET, 16 +%ifidn %%ENC_DEC, DEC + vmovdqa reg(i), %%T1 +%endif + ;; Prepare ciphertext for GHASH computations + vpshufb reg(i), [rel SHUF_MASK] + + ;; Hash all of the data +%assign i (8-%%num_initial_blocks) +%assign j (9-%%num_initial_blocks) +%assign k (%%num_initial_blocks) +%assign last_block_to_hash 0 + +%if(%%num_initial_blocks>last_block_to_hash) + ;; Hash in AES state + vpxor %%T2, reg(j) + + ;; T2 - incoming AAD hash + ;; reg(i) holds ciphertext + ;; T5 - hash key + ;; T6 - updated xor + ;; reg(1)/xmm1 should now be available for tmp use + vmovdqu %%T5, [%%GDATA_KEY + HashKey_ %+ k] + vpclmulqdq %%T1, %%T2, %%T5, 0x11 ; %%T4 = a1*b1 + vpclmulqdq %%T4, %%T2, %%T5, 0x00 ; %%T4 = a0*b0 + vpclmulqdq %%T6, %%T2, %%T5, 0x01 ; %%T6 = a1*b0 + vpclmulqdq %%T5, %%T2, %%T5, 0x10 ; %%T5 = a0*b1 + vpxor %%T6, %%T6, %%T5 +%endif + +%assign i (i+1) +%assign j (j+1) +%assign k (k-1) +%assign rep_count (%%num_initial_blocks-1) +%rep rep_count + + vmovdqu %%T5, [%%GDATA_KEY + HashKey_ %+ k] + vpclmulqdq %%T3, reg(j), %%T5, 0x11 + vpxor %%T1, %%T1, %%T3 + + vpclmulqdq %%T3, reg(j), %%T5, 0x00 + vpxor %%T4, %%T4, %%T3 + + vpclmulqdq %%T3, reg(j), %%T5, 0x01 + vpxor %%T6, %%T6, %%T3 + + vpclmulqdq %%T3, reg(j), %%T5, 0x10 + vpxor %%T6, %%T6, %%T3 + +%assign i (i+1) +%assign j (j+1) +%assign k (k-1) +%endrep + + ;; Record that a reduction is needed + mov r12, 1 + + jmp %%_small_initial_compute_hash + + +%endif ; %if %%num_initial_blocks < 8 + +%%_small_initial_partial_block: + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;; Handle ghash for a <16B final block +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + + ;; In this case if it's a single call to encrypt we can + ;; hash all of the data but if it's an init / update / finalize + ;; series of call we need to leave the last block if it's + ;; less than a full block of data. + + mov [%%GDATA_CTX + PBlockLen], %%LENGTH + vmovdqu [%%GDATA_CTX + PBlockEncKey], reg(i) + ;; Handle a partial final block + ;; GDATA, KEY, T1, T2 + ;; r13 - length + ;; LT16 - indicates type of read and that the buffer is less than 16 bytes long + ;; NOTE: could be replaced with %%LENGTH but at this point + ;; %%LENGTH is always less than 16. + ;; No PLAIN_CYPH_LEN argument available in this macro. + ENCRYPT_FINAL_PARTIAL_BLOCK reg(i), %%T1, %%CYPH_PLAIN_OUT, %%PLAIN_CYPH_IN, LT16, %%ENC_DEC, %%DATA_OFFSET + vpshufb reg(i), [rel SHUF_MASK] + +%ifidn %%INSTANCE_TYPE, multi_call +%assign i (8-%%num_initial_blocks) +%assign j (9-%%num_initial_blocks) +%assign k (%%num_initial_blocks-1) +%assign last_block_to_hash 1 +%else +%assign i (8-%%num_initial_blocks) +%assign j (9-%%num_initial_blocks) +%assign k (%%num_initial_blocks) +%assign last_block_to_hash 0 +%endif + +%if(%%num_initial_blocks>last_block_to_hash) + ;; Record that a reduction is needed + mov r12, 1 + ;; Hash in AES state + vpxor %%T2, reg(j) + + ;; T2 - incoming AAD hash + ;; reg(i) holds ciphertext + ;; T5 - hash key + ;; T6 - updated xor + ;; reg(1)/xmm1 should now be available for tmp use + vmovdqu %%T5, [%%GDATA_KEY + HashKey_ %+ k] + vpclmulqdq %%T1, %%T2, %%T5, 0x11 ; %%T4 = a1*b1 + vpclmulqdq %%T4, %%T2, %%T5, 0x00 ; %%T4 = a0*b0 + vpclmulqdq %%T6, %%T2, %%T5, 0x01 ; %%T6 = a1*b0 + vpclmulqdq %%T5, %%T2, %%T5, 0x10 ; %%T5 = a0*b1 + vpxor %%T6, %%T6, %%T5 +%else + ;; Record that a reduction is not needed - + ;; In this case no hashes are computed because there + ;; is only one initial block and it is < 16B in length. + xor r12, r12 +%endif + +%assign i (i+1) +%assign j (j+1) +%assign k (k-1) +%ifidn %%INSTANCE_TYPE, multi_call +%assign rep_count (%%num_initial_blocks-2) +%%_multi_call_hash: +%else +%assign rep_count (%%num_initial_blocks-1) +%endif + +%if rep_count < 0 + ;; fix for negative rep_count +%assign rep_count 0 +%endif + +%rep rep_count + + vmovdqu %%T5, [%%GDATA_KEY + HashKey_ %+ k] + vpclmulqdq %%T3, reg(j), %%T5, 0x11 + vpxor %%T1, %%T1, %%T3 + + vpclmulqdq %%T3, reg(j), %%T5, 0x00 + vpxor %%T4, %%T4, %%T3 + + vpclmulqdq %%T3, reg(j), %%T5, 0x01 + vpxor %%T6, %%T6, %%T3 + + vpclmulqdq %%T3, reg(j), %%T5, 0x10 + vpxor %%T6, %%T6, %%T3 + +%assign i (i+1) +%assign j (j+1) +%assign k (k-1) +%endrep + +%%_small_initial_compute_hash: + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;; Ghash reduction +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +%if(%%num_initial_blocks=1) +%ifidn %%INSTANCE_TYPE, multi_call + ;; We only need to check if a reduction is needed if + ;; initial_blocks == 1 and init/update/final is being used. + ;; In this case we may just have a partial block, and that + ;; gets hashed in finalize. + ;; cmp r12, 0 + or r12, r12 + je %%_no_reduction_needed +%endif +%endif + + vpsrldq %%T3, %%T6, 8 ; shift-R %%T2 2 DWs + vpslldq %%T6, %%T6, 8 ; shift-L %%T3 2 DWs + vpxor %%T1, %%T1, %%T3 ; accumulate the results in %%T1:%%T4 + vpxor %%T4, %%T6, %%T4 + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; First phase of the reduction + vmovdqu %%T3, [rel POLY2] + + vpclmulqdq %%T2, %%T3, %%T4, 0x01 + ;; shift-L xmm2 2 DWs + vpslldq %%T2, %%T2, 8 + vpxor %%T4, %%T4, %%T2 + + ;; First phase of the reduction complete + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; Second phase of the reduction + + vpclmulqdq %%T2, %%T3, %%T4, 0x00 + ;; Shift-R xmm2 1 DW (Shift-R only 1-DW to obtain 2-DWs shift-R) + vpsrldq %%T2, %%T2, 4 + + vpclmulqdq %%T4, %%T3, %%T4, 0x10 + ;; Shift-L xmm0 1 DW (Shift-L 1-DW to obtain result with no shifts) + vpslldq %%T4, %%T4, 4 + + vpxor %%T4, %%T4, %%T2 + ;; Second phase of the reduction complete + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + vpxor %%T3, %%T1, %%T4 + +%ifidn %%INSTANCE_TYPE, multi_call + ;; If using init/update/finalize, we need to xor any partial block data + ;; into the hash. +%if %%num_initial_blocks > 1 + ;; NOTE: for %%num_initial_blocks = 0 the xor never takes place +%if %%num_initial_blocks != 8 + ;; NOTE: for %%num_initial_blocks = 8, %%LENGTH, stored in [PBlockLen] is never zero + cmp qword [%%GDATA_CTX + PBlockLen], 0 + je %%_no_partial_block_xor +%endif ; %%num_initial_blocks != 8 + vpxor %%T3, %%T3, reg(8) +%%_no_partial_block_xor: +%endif ; %%num_initial_blocks > 1 +%endif ; %%INSTANCE_TYPE, multi_call + +%if(%%num_initial_blocks=1) +%ifidn %%INSTANCE_TYPE, multi_call + ;; NOTE: %%_no_reduction_needed case only valid for + ;; multi_call with initial_blocks = 1. + ;; Look for comment above around '_no_reduction_needed' + ;; The jmp below is obsolete as the code will fall through. + + ;; The result is in %%T3 + jmp %%_after_reduction + +%%_no_reduction_needed: + ;; The hash should end up in T3. The only way we should get here is if + ;; there is a partial block of data, so xor that into the hash. + vpxor %%T3, %%T2, reg(8) +%endif ; %%INSTANCE_TYPE = multi_call +%endif ; %%num_initial_blocks=1 + +%%_after_reduction: + ;; Final hash is now in T3 + +%endmacro ; INITIAL_BLOCKS_PARTIAL + + + +; encrypt 8 blocks at a time +; ghash the 8 previously encrypted ciphertext blocks +; %%GDATA (KEY), %%CYPH_PLAIN_OUT, %%PLAIN_CYPH_IN are used as pointers only, not modified +; %%DATA_OFFSET is the data offset value +%macro GHASH_8_ENCRYPT_8_PARALLEL 23 +%define %%GDATA %1 +%define %%CYPH_PLAIN_OUT %2 +%define %%PLAIN_CYPH_IN %3 +%define %%DATA_OFFSET %4 +%define %%T1 %5 +%define %%T2 %6 +%define %%T3 %7 +%define %%T4 %8 +%define %%T5 %9 +%define %%T6 %10 +%define %%CTR %11 +%define %%XMM1 %12 +%define %%XMM2 %13 +%define %%XMM3 %14 +%define %%XMM4 %15 +%define %%XMM5 %16 +%define %%XMM6 %17 +%define %%XMM7 %18 +%define %%XMM8 %19 +%define %%T7 %20 +%define %%loop_idx %21 +%define %%ENC_DEC %22 +%define %%FULL_PARTIAL %23 + + vmovdqa %%T2, %%XMM1 + vmovdqu [rsp + TMP2], %%XMM2 + vmovdqu [rsp + TMP3], %%XMM3 + vmovdqu [rsp + TMP4], %%XMM4 + vmovdqu [rsp + TMP5], %%XMM5 + vmovdqu [rsp + TMP6], %%XMM6 + vmovdqu [rsp + TMP7], %%XMM7 + vmovdqu [rsp + TMP8], %%XMM8 + +%ifidn %%loop_idx, in_order + vpaddd %%XMM1, %%CTR, [rel ONE] ; INCR CNT + vmovdqu %%T5, [rel TWO] + vpaddd %%XMM2, %%CTR, %%T5 + vpaddd %%XMM3, %%XMM1, %%T5 + vpaddd %%XMM4, %%XMM2, %%T5 + vpaddd %%XMM5, %%XMM3, %%T5 + vpaddd %%XMM6, %%XMM4, %%T5 + vpaddd %%XMM7, %%XMM5, %%T5 + vpaddd %%XMM8, %%XMM6, %%T5 + vmovdqa %%CTR, %%XMM8 + + vmovdqu %%T5, [rel SHUF_MASK] + vpshufb %%XMM1, %%T5 ; perform a 16Byte swap + vpshufb %%XMM2, %%T5 ; perform a 16Byte swap + vpshufb %%XMM3, %%T5 ; perform a 16Byte swap + vpshufb %%XMM4, %%T5 ; perform a 16Byte swap + vpshufb %%XMM5, %%T5 ; perform a 16Byte swap + vpshufb %%XMM6, %%T5 ; perform a 16Byte swap + vpshufb %%XMM7, %%T5 ; perform a 16Byte swap + vpshufb %%XMM8, %%T5 ; perform a 16Byte swap +%else + vpaddd %%XMM1, %%CTR, [rel ONEf] ; INCR CNT + vmovdqu %%T5, [rel TWOf] + vpaddd %%XMM2, %%CTR, %%T5 + vpaddd %%XMM3, %%XMM1, %%T5 + vpaddd %%XMM4, %%XMM2, %%T5 + vpaddd %%XMM5, %%XMM3, %%T5 + vpaddd %%XMM6, %%XMM4, %%T5 + vpaddd %%XMM7, %%XMM5, %%T5 + vpaddd %%XMM8, %%XMM6, %%T5 + vmovdqa %%CTR, %%XMM8 +%endif + + + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + + vmovdqu %%T1, [%%GDATA + 16*0] + vpxor %%XMM1, %%XMM1, %%T1 + vpxor %%XMM2, %%XMM2, %%T1 + vpxor %%XMM3, %%XMM3, %%T1 + vpxor %%XMM4, %%XMM4, %%T1 + vpxor %%XMM5, %%XMM5, %%T1 + vpxor %%XMM6, %%XMM6, %%T1 + vpxor %%XMM7, %%XMM7, %%T1 + vpxor %%XMM8, %%XMM8, %%T1 + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + + vmovdqu %%T1, [%%GDATA + 16*1] + vaesenc %%XMM1, %%T1 + vaesenc %%XMM2, %%T1 + vaesenc %%XMM3, %%T1 + vaesenc %%XMM4, %%T1 + vaesenc %%XMM5, %%T1 + vaesenc %%XMM6, %%T1 + vaesenc %%XMM7, %%T1 + vaesenc %%XMM8, %%T1 + + + vmovdqu %%T1, [%%GDATA + 16*2] + vaesenc %%XMM1, %%T1 + vaesenc %%XMM2, %%T1 + vaesenc %%XMM3, %%T1 + vaesenc %%XMM4, %%T1 + vaesenc %%XMM5, %%T1 + vaesenc %%XMM6, %%T1 + vaesenc %%XMM7, %%T1 + vaesenc %%XMM8, %%T1 + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + + vmovdqu %%T5, [%%GDATA + HashKey_8] + vpclmulqdq %%T4, %%T2, %%T5, 0x11 ; %%T4 = a1*b1 + vpclmulqdq %%T7, %%T2, %%T5, 0x00 ; %%T7 = a0*b0 + vpclmulqdq %%T6, %%T2, %%T5, 0x01 ; %%T6 = a1*b0 + vpclmulqdq %%T5, %%T2, %%T5, 0x10 ; %%T5 = a0*b1 + vpxor %%T6, %%T6, %%T5 + + vmovdqu %%T1, [%%GDATA + 16*3] + vaesenc %%XMM1, %%T1 + vaesenc %%XMM2, %%T1 + vaesenc %%XMM3, %%T1 + vaesenc %%XMM4, %%T1 + vaesenc %%XMM5, %%T1 + vaesenc %%XMM6, %%T1 + vaesenc %%XMM7, %%T1 + vaesenc %%XMM8, %%T1 + + vmovdqu %%T1, [rsp + TMP2] + vmovdqu %%T5, [%%GDATA + HashKey_7] + vpclmulqdq %%T3, %%T1, %%T5, 0x11 + vpxor %%T4, %%T4, %%T3 + + vpclmulqdq %%T3, %%T1, %%T5, 0x00 + vpxor %%T7, %%T7, %%T3 + + vpclmulqdq %%T3, %%T1, %%T5, 0x01 + vpxor %%T6, %%T6, %%T3 + + vpclmulqdq %%T3, %%T1, %%T5, 0x10 + vpxor %%T6, %%T6, %%T3 + + vmovdqu %%T1, [%%GDATA + 16*4] + vaesenc %%XMM1, %%T1 + vaesenc %%XMM2, %%T1 + vaesenc %%XMM3, %%T1 + vaesenc %%XMM4, %%T1 + vaesenc %%XMM5, %%T1 + vaesenc %%XMM6, %%T1 + vaesenc %%XMM7, %%T1 + vaesenc %%XMM8, %%T1 + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + vmovdqu %%T1, [rsp + TMP3] + vmovdqu %%T5, [%%GDATA + HashKey_6] + vpclmulqdq %%T3, %%T1, %%T5, 0x11 + vpxor %%T4, %%T4, %%T3 + + vpclmulqdq %%T3, %%T1, %%T5, 0x00 + vpxor %%T7, %%T7, %%T3 + + vpclmulqdq %%T3, %%T1, %%T5, 0x01 + vpxor %%T6, %%T6, %%T3 + + vpclmulqdq %%T3, %%T1, %%T5, 0x10 + vpxor %%T6, %%T6, %%T3 + + vmovdqu %%T1, [%%GDATA + 16*5] + vaesenc %%XMM1, %%T1 + vaesenc %%XMM2, %%T1 + vaesenc %%XMM3, %%T1 + vaesenc %%XMM4, %%T1 + vaesenc %%XMM5, %%T1 + vaesenc %%XMM6, %%T1 + vaesenc %%XMM7, %%T1 + vaesenc %%XMM8, %%T1 + + + vmovdqu %%T1, [rsp + TMP4] + vmovdqu %%T5, [%%GDATA + HashKey_5] + vpclmulqdq %%T3, %%T1, %%T5, 0x11 + vpxor %%T4, %%T4, %%T3 + + vpclmulqdq %%T3, %%T1, %%T5, 0x00 + vpxor %%T7, %%T7, %%T3 + + vpclmulqdq %%T3, %%T1, %%T5, 0x01 + vpxor %%T6, %%T6, %%T3 + + vpclmulqdq %%T3, %%T1, %%T5, 0x10 + vpxor %%T6, %%T6, %%T3 + + vmovdqu %%T1, [%%GDATA + 16*6] + vaesenc %%XMM1, %%T1 + vaesenc %%XMM2, %%T1 + vaesenc %%XMM3, %%T1 + vaesenc %%XMM4, %%T1 + vaesenc %%XMM5, %%T1 + vaesenc %%XMM6, %%T1 + vaesenc %%XMM7, %%T1 + vaesenc %%XMM8, %%T1 + + vmovdqu %%T1, [rsp + TMP5] + vmovdqu %%T5, [%%GDATA + HashKey_4] + vpclmulqdq %%T3, %%T1, %%T5, 0x11 + vpxor %%T4, %%T4, %%T3 + + vpclmulqdq %%T3, %%T1, %%T5, 0x00 + vpxor %%T7, %%T7, %%T3 + + vpclmulqdq %%T3, %%T1, %%T5, 0x01 + vpxor %%T6, %%T6, %%T3 + + vpclmulqdq %%T3, %%T1, %%T5, 0x10 + vpxor %%T6, %%T6, %%T3 + + vmovdqu %%T1, [%%GDATA + 16*7] + vaesenc %%XMM1, %%T1 + vaesenc %%XMM2, %%T1 + vaesenc %%XMM3, %%T1 + vaesenc %%XMM4, %%T1 + vaesenc %%XMM5, %%T1 + vaesenc %%XMM6, %%T1 + vaesenc %%XMM7, %%T1 + vaesenc %%XMM8, %%T1 + + vmovdqu %%T1, [rsp + TMP6] + vmovdqu %%T5, [%%GDATA + HashKey_3] + vpclmulqdq %%T3, %%T1, %%T5, 0x11 + vpxor %%T4, %%T4, %%T3 + + vpclmulqdq %%T3, %%T1, %%T5, 0x00 + vpxor %%T7, %%T7, %%T3 + + vpclmulqdq %%T3, %%T1, %%T5, 0x01 + vpxor %%T6, %%T6, %%T3 + + vpclmulqdq %%T3, %%T1, %%T5, 0x10 + vpxor %%T6, %%T6, %%T3 + + vmovdqu %%T1, [%%GDATA + 16*8] + vaesenc %%XMM1, %%T1 + vaesenc %%XMM2, %%T1 + vaesenc %%XMM3, %%T1 + vaesenc %%XMM4, %%T1 + vaesenc %%XMM5, %%T1 + vaesenc %%XMM6, %%T1 + vaesenc %%XMM7, %%T1 + vaesenc %%XMM8, %%T1 + + vmovdqu %%T1, [rsp + TMP7] + vmovdqu %%T5, [%%GDATA + HashKey_2] + vpclmulqdq %%T3, %%T1, %%T5, 0x11 + vpxor %%T4, %%T4, %%T3 + + vpclmulqdq %%T3, %%T1, %%T5, 0x00 + vpxor %%T7, %%T7, %%T3 + + vpclmulqdq %%T3, %%T1, %%T5, 0x01 + vpxor %%T6, %%T6, %%T3 + + vpclmulqdq %%T3, %%T1, %%T5, 0x10 + vpxor %%T6, %%T6, %%T3 + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + + vmovdqu %%T5, [%%GDATA + 16*9] + vaesenc %%XMM1, %%T5 + vaesenc %%XMM2, %%T5 + vaesenc %%XMM3, %%T5 + vaesenc %%XMM4, %%T5 + vaesenc %%XMM5, %%T5 + vaesenc %%XMM6, %%T5 + vaesenc %%XMM7, %%T5 + vaesenc %%XMM8, %%T5 + + vmovdqu %%T1, [rsp + TMP8] + vmovdqu %%T5, [%%GDATA + HashKey] + + + vpclmulqdq %%T3, %%T1, %%T5, 0x00 + vpxor %%T7, %%T7, %%T3 + + vpclmulqdq %%T3, %%T1, %%T5, 0x01 + vpxor %%T6, %%T6, %%T3 + + vpclmulqdq %%T3, %%T1, %%T5, 0x10 + vpxor %%T6, %%T6, %%T3 + + vpclmulqdq %%T3, %%T1, %%T5, 0x11 + vpxor %%T1, %%T4, %%T3 + + + vmovdqu %%T5, [%%GDATA + 16*10] + %ifndef GCM128_MODE ; GCM192 or GCM256 + vaesenc %%XMM1, %%T5 + vaesenc %%XMM2, %%T5 + vaesenc %%XMM3, %%T5 + vaesenc %%XMM4, %%T5 + vaesenc %%XMM5, %%T5 + vaesenc %%XMM6, %%T5 + vaesenc %%XMM7, %%T5 + vaesenc %%XMM8, %%T5 + + vmovdqu %%T5, [%%GDATA + 16*11] + vaesenc %%XMM1, %%T5 + vaesenc %%XMM2, %%T5 + vaesenc %%XMM3, %%T5 + vaesenc %%XMM4, %%T5 + vaesenc %%XMM5, %%T5 + vaesenc %%XMM6, %%T5 + vaesenc %%XMM7, %%T5 + vaesenc %%XMM8, %%T5 + + vmovdqu %%T5, [%%GDATA + 16*12] +%endif +%ifdef GCM256_MODE + vaesenc %%XMM1, %%T5 + vaesenc %%XMM2, %%T5 + vaesenc %%XMM3, %%T5 + vaesenc %%XMM4, %%T5 + vaesenc %%XMM5, %%T5 + vaesenc %%XMM6, %%T5 + vaesenc %%XMM7, %%T5 + vaesenc %%XMM8, %%T5 + + vmovdqu %%T5, [%%GDATA + 16*13] + vaesenc %%XMM1, %%T5 + vaesenc %%XMM2, %%T5 + vaesenc %%XMM3, %%T5 + vaesenc %%XMM4, %%T5 + vaesenc %%XMM5, %%T5 + vaesenc %%XMM6, %%T5 + vaesenc %%XMM7, %%T5 + vaesenc %%XMM8, %%T5 + + vmovdqu %%T5, [%%GDATA + 16*14] +%endif ; GCM256 + +%assign i 0 +%assign j 1 +%rep 8 + + ;; SNP TBD: This is pretty ugly - consider whether just XORing the + ;; data in after vaesenclast is simpler and performant. Would + ;; also have to ripple it through partial block and ghash_mul_8. +%ifidn %%FULL_PARTIAL, full + %ifdef NT_LD + VXLDR %%T2, [%%PLAIN_CYPH_IN+%%DATA_OFFSET+16*i] + vpxor %%T2, %%T2, %%T5 + %else + vpxor %%T2, %%T5, [%%PLAIN_CYPH_IN+%%DATA_OFFSET+16*i] + %endif + + %ifidn %%ENC_DEC, ENC + vaesenclast reg(j), reg(j), %%T2 + %else + vaesenclast %%T3, reg(j), %%T2 + vpxor reg(j), %%T2, %%T5 + VXSTR [%%CYPH_PLAIN_OUT+%%DATA_OFFSET+16*i], %%T3 + %endif + +%else + ; Don't read the final data during partial block processing + %ifdef NT_LD + %if (i<7) + VXLDR %%T2, [%%PLAIN_CYPH_IN+%%DATA_OFFSET+16*i] + vpxor %%T2, %%T2, %%T5 + %else + ;; Stage the key directly in T2 rather than hash it with plaintext + vmovdqu %%T2, %%T5 + %endif + %else + %if (i<7) + vpxor %%T2, %%T5, [%%PLAIN_CYPH_IN+%%DATA_OFFSET+16*i] + %else + ;; Stage the key directly in T2 rather than hash it with plaintext + vmovdqu %%T2, %%T5 + %endif + %endif + + %ifidn %%ENC_DEC, ENC + vaesenclast reg(j), reg(j), %%T2 + %else + %if (i<7) + vaesenclast %%T3, reg(j), %%T2 + vpxor reg(j), %%T2, %%T5 + ;; Do not read the data since it could fault + VXSTR [%%CYPH_PLAIN_OUT+%%DATA_OFFSET+16*i], %%T3 + %else + vaesenclast reg(j), reg(j), %%T2 + %endif + %endif +%endif + +%assign i (i+1) +%assign j (j+1) +%endrep + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + + + vpslldq %%T3, %%T6, 8 ; shift-L %%T3 2 DWs + vpsrldq %%T6, %%T6, 8 ; shift-R %%T2 2 DWs + vpxor %%T7, %%T7, %%T3 + vpxor %%T1, %%T1, %%T6 ; accumulate the results in %%T1:%%T7 + + + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;first phase of the reduction + vmovdqu %%T3, [rel POLY2] + + vpclmulqdq %%T2, %%T3, %%T7, 0x01 + vpslldq %%T2, %%T2, 8 ; shift-L xmm2 2 DWs + + vpxor %%T7, %%T7, %%T2 ; first phase of the reduction complete + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + + %ifidn %%ENC_DEC, ENC + ; Write to the Ciphertext buffer + VXSTR [%%CYPH_PLAIN_OUT+%%DATA_OFFSET+16*0], %%XMM1 + VXSTR [%%CYPH_PLAIN_OUT+%%DATA_OFFSET+16*1], %%XMM2 + VXSTR [%%CYPH_PLAIN_OUT+%%DATA_OFFSET+16*2], %%XMM3 + VXSTR [%%CYPH_PLAIN_OUT+%%DATA_OFFSET+16*3], %%XMM4 + VXSTR [%%CYPH_PLAIN_OUT+%%DATA_OFFSET+16*4], %%XMM5 + VXSTR [%%CYPH_PLAIN_OUT+%%DATA_OFFSET+16*5], %%XMM6 + VXSTR [%%CYPH_PLAIN_OUT+%%DATA_OFFSET+16*6], %%XMM7 + %ifidn %%FULL_PARTIAL, full + ;; Avoid writing past the buffer if handling a partial block + VXSTR [%%CYPH_PLAIN_OUT+%%DATA_OFFSET+16*7], %%XMM8 + %endif + %endif + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;second phase of the reduction + vpclmulqdq %%T2, %%T3, %%T7, 0x00 + vpsrldq %%T2, %%T2, 4 ; shift-R xmm2 1 DW (Shift-R only 1-DW to obtain 2-DWs shift-R) + + vpclmulqdq %%T4, %%T3, %%T7, 0x10 + vpslldq %%T4, %%T4, 4 ; shift-L xmm0 1 DW (Shift-L 1-DW to obtain result with no shifts) + + vpxor %%T4, %%T4, %%T2 ; second phase of the reduction complete + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + vpxor %%T1, %%T1, %%T4 ; the result is in %%T1 + + vpshufb %%XMM1, [rel SHUF_MASK] ; perform a 16Byte swap + vpshufb %%XMM2, [rel SHUF_MASK] ; perform a 16Byte swap + vpshufb %%XMM3, [rel SHUF_MASK] ; perform a 16Byte swap + vpshufb %%XMM4, [rel SHUF_MASK] ; perform a 16Byte swap + vpshufb %%XMM5, [rel SHUF_MASK] ; perform a 16Byte swap + vpshufb %%XMM6, [rel SHUF_MASK] ; perform a 16Byte swap + vpshufb %%XMM7, [rel SHUF_MASK] ; perform a 16Byte swap + vpshufb %%XMM8, [rel SHUF_MASK] ; perform a 16Byte swap + + + vpxor %%XMM1, %%T1 + + +%endmacro ; GHASH_8_ENCRYPT_8_PARALLEL + + +; GHASH the last 4 ciphertext blocks. +%macro GHASH_LAST_8 16 +%define %%GDATA %1 +%define %%T1 %2 +%define %%T2 %3 +%define %%T3 %4 +%define %%T4 %5 +%define %%T5 %6 +%define %%T6 %7 +%define %%T7 %8 +%define %%XMM1 %9 +%define %%XMM2 %10 +%define %%XMM3 %11 +%define %%XMM4 %12 +%define %%XMM5 %13 +%define %%XMM6 %14 +%define %%XMM7 %15 +%define %%XMM8 %16 + + ;; Karatsuba Method + + vmovdqu %%T5, [%%GDATA + HashKey_8] + + vpshufd %%T2, %%XMM1, 01001110b + vpshufd %%T3, %%T5, 01001110b + vpxor %%T2, %%T2, %%XMM1 + vpxor %%T3, %%T3, %%T5 + + vpclmulqdq %%T6, %%XMM1, %%T5, 0x11 + vpclmulqdq %%T7, %%XMM1, %%T5, 0x00 + + vpclmulqdq %%XMM1, %%T2, %%T3, 0x00 + + ;;;;;;;;;;;;;;;;;;;;;; + + vmovdqu %%T5, [%%GDATA + HashKey_7] + vpshufd %%T2, %%XMM2, 01001110b + vpshufd %%T3, %%T5, 01001110b + vpxor %%T2, %%T2, %%XMM2 + vpxor %%T3, %%T3, %%T5 + + vpclmulqdq %%T4, %%XMM2, %%T5, 0x11 + vpxor %%T6, %%T6, %%T4 + + vpclmulqdq %%T4, %%XMM2, %%T5, 0x00 + vpxor %%T7, %%T7, %%T4 + + vpclmulqdq %%T2, %%T2, %%T3, 0x00 + + vpxor %%XMM1, %%XMM1, %%T2 + + ;;;;;;;;;;;;;;;;;;;;;; + + vmovdqu %%T5, [%%GDATA + HashKey_6] + vpshufd %%T2, %%XMM3, 01001110b + vpshufd %%T3, %%T5, 01001110b + vpxor %%T2, %%T2, %%XMM3 + vpxor %%T3, %%T3, %%T5 + + vpclmulqdq %%T4, %%XMM3, %%T5, 0x11 + vpxor %%T6, %%T6, %%T4 + + vpclmulqdq %%T4, %%XMM3, %%T5, 0x00 + vpxor %%T7, %%T7, %%T4 + + vpclmulqdq %%T2, %%T2, %%T3, 0x00 + + vpxor %%XMM1, %%XMM1, %%T2 + + ;;;;;;;;;;;;;;;;;;;;;; + + vmovdqu %%T5, [%%GDATA + HashKey_5] + vpshufd %%T2, %%XMM4, 01001110b + vpshufd %%T3, %%T5, 01001110b + vpxor %%T2, %%T2, %%XMM4 + vpxor %%T3, %%T3, %%T5 + + vpclmulqdq %%T4, %%XMM4, %%T5, 0x11 + vpxor %%T6, %%T6, %%T4 + + vpclmulqdq %%T4, %%XMM4, %%T5, 0x00 + vpxor %%T7, %%T7, %%T4 + + vpclmulqdq %%T2, %%T2, %%T3, 0x00 + + vpxor %%XMM1, %%XMM1, %%T2 + + ;;;;;;;;;;;;;;;;;;;;;; + + vmovdqu %%T5, [%%GDATA + HashKey_4] + vpshufd %%T2, %%XMM5, 01001110b + vpshufd %%T3, %%T5, 01001110b + vpxor %%T2, %%T2, %%XMM5 + vpxor %%T3, %%T3, %%T5 + + vpclmulqdq %%T4, %%XMM5, %%T5, 0x11 + vpxor %%T6, %%T6, %%T4 + + vpclmulqdq %%T4, %%XMM5, %%T5, 0x00 + vpxor %%T7, %%T7, %%T4 + + vpclmulqdq %%T2, %%T2, %%T3, 0x00 + + vpxor %%XMM1, %%XMM1, %%T2 + + ;;;;;;;;;;;;;;;;;;;;;; + + vmovdqu %%T5, [%%GDATA + HashKey_3] + vpshufd %%T2, %%XMM6, 01001110b + vpshufd %%T3, %%T5, 01001110b + vpxor %%T2, %%T2, %%XMM6 + vpxor %%T3, %%T3, %%T5 + + vpclmulqdq %%T4, %%XMM6, %%T5, 0x11 + vpxor %%T6, %%T6, %%T4 + + vpclmulqdq %%T4, %%XMM6, %%T5, 0x00 + vpxor %%T7, %%T7, %%T4 + + vpclmulqdq %%T2, %%T2, %%T3, 0x00 + + vpxor %%XMM1, %%XMM1, %%T2 + + ;;;;;;;;;;;;;;;;;;;;;; + + vmovdqu %%T5, [%%GDATA + HashKey_2] + vpshufd %%T2, %%XMM7, 01001110b + vpshufd %%T3, %%T5, 01001110b + vpxor %%T2, %%T2, %%XMM7 + vpxor %%T3, %%T3, %%T5 + + vpclmulqdq %%T4, %%XMM7, %%T5, 0x11 + vpxor %%T6, %%T6, %%T4 + + vpclmulqdq %%T4, %%XMM7, %%T5, 0x00 + vpxor %%T7, %%T7, %%T4 + + vpclmulqdq %%T2, %%T2, %%T3, 0x00 + + vpxor %%XMM1, %%XMM1, %%T2 + + ;;;;;;;;;;;;;;;;;;;;;; + + vmovdqu %%T5, [%%GDATA + HashKey] + vpshufd %%T2, %%XMM8, 01001110b + vpshufd %%T3, %%T5, 01001110b + vpxor %%T2, %%T2, %%XMM8 + vpxor %%T3, %%T3, %%T5 + + vpclmulqdq %%T4, %%XMM8, %%T5, 0x11 + vpxor %%T6, %%T6, %%T4 + + vpclmulqdq %%T4, %%XMM8, %%T5, 0x00 + vpxor %%T7, %%T7, %%T4 + + vpclmulqdq %%T2, %%T2, %%T3, 0x00 + + vpxor %%XMM1, %%XMM1, %%T2 + vpxor %%XMM1, %%XMM1, %%T6 + vpxor %%T2, %%XMM1, %%T7 + + + + + vpslldq %%T4, %%T2, 8 + vpsrldq %%T2, %%T2, 8 + + vpxor %%T7, %%T7, %%T4 + vpxor %%T6, %%T6, %%T2 ; <%%T6:%%T7> holds the result of the accumulated carry-less multiplications + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;first phase of the reduction + vmovdqu %%T3, [rel POLY2] + + vpclmulqdq %%T2, %%T3, %%T7, 0x01 + vpslldq %%T2, %%T2, 8 ; shift-L xmm2 2 DWs + + vpxor %%T7, %%T7, %%T2 ; first phase of the reduction complete + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + + + ;second phase of the reduction + vpclmulqdq %%T2, %%T3, %%T7, 0x00 + vpsrldq %%T2, %%T2, 4 ; shift-R %%T2 1 DW (Shift-R only 1-DW to obtain 2-DWs shift-R) + + vpclmulqdq %%T4, %%T3, %%T7, 0x10 + vpslldq %%T4, %%T4, 4 ; shift-L %%T4 1 DW (Shift-L 1-DW to obtain result with no shifts) + + vpxor %%T4, %%T4, %%T2 ; second phase of the reduction complete + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + vpxor %%T6, %%T6, %%T4 ; the result is in %%T6 +%endmacro + + +; GHASH the last 4 ciphertext blocks. +%macro GHASH_LAST_7 15 +%define %%GDATA %1 +%define %%T1 %2 +%define %%T2 %3 +%define %%T3 %4 +%define %%T4 %5 +%define %%T5 %6 +%define %%T6 %7 +%define %%T7 %8 +%define %%XMM1 %9 +%define %%XMM2 %10 +%define %%XMM3 %11 +%define %%XMM4 %12 +%define %%XMM5 %13 +%define %%XMM6 %14 +%define %%XMM7 %15 + + ;; Karatsuba Method + + vmovdqu %%T5, [%%GDATA + HashKey_7] + + vpshufd %%T2, %%XMM1, 01001110b + vpshufd %%T3, %%T5, 01001110b + vpxor %%T2, %%T2, %%XMM1 + vpxor %%T3, %%T3, %%T5 + + vpclmulqdq %%T6, %%XMM1, %%T5, 0x11 + vpclmulqdq %%T7, %%XMM1, %%T5, 0x00 + + vpclmulqdq %%XMM1, %%T2, %%T3, 0x00 + + ;;;;;;;;;;;;;;;;;;;;;; + + vmovdqu %%T5, [%%GDATA + HashKey_6] + vpshufd %%T2, %%XMM2, 01001110b + vpshufd %%T3, %%T5, 01001110b + vpxor %%T2, %%T2, %%XMM2 + vpxor %%T3, %%T3, %%T5 + + vpclmulqdq %%T4, %%XMM2, %%T5, 0x11 + vpxor %%T6, %%T6, %%T4 + + vpclmulqdq %%T4, %%XMM2, %%T5, 0x00 + vpxor %%T7, %%T7, %%T4 + + vpclmulqdq %%T2, %%T2, %%T3, 0x00 + + vpxor %%XMM1, %%XMM1, %%T2 + + ;;;;;;;;;;;;;;;;;;;;;; + + vmovdqu %%T5, [%%GDATA + HashKey_5] + vpshufd %%T2, %%XMM3, 01001110b + vpshufd %%T3, %%T5, 01001110b + vpxor %%T2, %%T2, %%XMM3 + vpxor %%T3, %%T3, %%T5 + + vpclmulqdq %%T4, %%XMM3, %%T5, 0x11 + vpxor %%T6, %%T6, %%T4 + + vpclmulqdq %%T4, %%XMM3, %%T5, 0x00 + vpxor %%T7, %%T7, %%T4 + + vpclmulqdq %%T2, %%T2, %%T3, 0x00 + + vpxor %%XMM1, %%XMM1, %%T2 + + ;;;;;;;;;;;;;;;;;;;;;; + + vmovdqu %%T5, [%%GDATA + HashKey_4] + vpshufd %%T2, %%XMM4, 01001110b + vpshufd %%T3, %%T5, 01001110b + vpxor %%T2, %%T2, %%XMM4 + vpxor %%T3, %%T3, %%T5 + + vpclmulqdq %%T4, %%XMM4, %%T5, 0x11 + vpxor %%T6, %%T6, %%T4 + + vpclmulqdq %%T4, %%XMM4, %%T5, 0x00 + vpxor %%T7, %%T7, %%T4 + + vpclmulqdq %%T2, %%T2, %%T3, 0x00 + + vpxor %%XMM1, %%XMM1, %%T2 + + ;;;;;;;;;;;;;;;;;;;;;; + + vmovdqu %%T5, [%%GDATA + HashKey_3] + vpshufd %%T2, %%XMM5, 01001110b + vpshufd %%T3, %%T5, 01001110b + vpxor %%T2, %%T2, %%XMM5 + vpxor %%T3, %%T3, %%T5 + + vpclmulqdq %%T4, %%XMM5, %%T5, 0x11 + vpxor %%T6, %%T6, %%T4 + + vpclmulqdq %%T4, %%XMM5, %%T5, 0x00 + vpxor %%T7, %%T7, %%T4 + + vpclmulqdq %%T2, %%T2, %%T3, 0x00 + + vpxor %%XMM1, %%XMM1, %%T2 + + ;;;;;;;;;;;;;;;;;;;;;; + + vmovdqu %%T5, [%%GDATA + HashKey_2] + vpshufd %%T2, %%XMM6, 01001110b + vpshufd %%T3, %%T5, 01001110b + vpxor %%T2, %%T2, %%XMM6 + vpxor %%T3, %%T3, %%T5 + + vpclmulqdq %%T4, %%XMM6, %%T5, 0x11 + vpxor %%T6, %%T6, %%T4 + + vpclmulqdq %%T4, %%XMM6, %%T5, 0x00 + vpxor %%T7, %%T7, %%T4 + + vpclmulqdq %%T2, %%T2, %%T3, 0x00 + + vpxor %%XMM1, %%XMM1, %%T2 + + ;;;;;;;;;;;;;;;;;;;;;; + + vmovdqu %%T5, [%%GDATA + HashKey_1] + vpshufd %%T2, %%XMM7, 01001110b + vpshufd %%T3, %%T5, 01001110b + vpxor %%T2, %%T2, %%XMM7 + vpxor %%T3, %%T3, %%T5 + + vpclmulqdq %%T4, %%XMM7, %%T5, 0x11 + vpxor %%T6, %%T6, %%T4 + + vpclmulqdq %%T4, %%XMM7, %%T5, 0x00 + vpxor %%T7, %%T7, %%T4 + + vpclmulqdq %%T2, %%T2, %%T3, 0x00 + + vpxor %%XMM1, %%XMM1, %%T2 + + ;;;;;;;;;;;;;;;;;;;;;; + + vpxor %%XMM1, %%XMM1, %%T6 + vpxor %%T2, %%XMM1, %%T7 + + + + + vpslldq %%T4, %%T2, 8 + vpsrldq %%T2, %%T2, 8 + + vpxor %%T7, %%T7, %%T4 + vpxor %%T6, %%T6, %%T2 ; <%%T6:%%T7> holds the result of the accumulated carry-less multiplications + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;first phase of the reduction + vmovdqu %%T3, [rel POLY2] + + vpclmulqdq %%T2, %%T3, %%T7, 0x01 + vpslldq %%T2, %%T2, 8 ; shift-L xmm2 2 DWs + + vpxor %%T7, %%T7, %%T2 ; first phase of the reduction complete + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + + + ;second phase of the reduction + vpclmulqdq %%T2, %%T3, %%T7, 0x00 + vpsrldq %%T2, %%T2, 4 ; shift-R %%T2 1 DW (Shift-R only 1-DW to obtain 2-DWs shift-R) + + vpclmulqdq %%T4, %%T3, %%T7, 0x10 + vpslldq %%T4, %%T4, 4 ; shift-L %%T4 1 DW (Shift-L 1-DW to obtain result with no shifts) + + vpxor %%T4, %%T4, %%T2 ; second phase of the reduction complete + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + vpxor %%T6, %%T6, %%T4 ; the result is in %%T6 +%endmacro + + + +;;; Handle encryption of the final partial block +;;; IN: +;;; r13 - Number of bytes to read +;;; MODIFIES: +;;; KEY - Key for encrypting the partial block +;;; SMASHES: +;;; rax, T1 +;;; Note: +;;; PLAIN_CYPH_LEN is unused at this stage. Previously: +;;; it was used to determine if buffer is big enough to do +;;; a 16 byte read & shift. +;;; 'LT16' is passed here only if buffer is known to be smaller +;;; than 16 bytes. +;;; Any other value passed here will result in 16 byte read +;;; code path. +%macro ENCRYPT_FINAL_PARTIAL_BLOCK 7 +%define %%KEY %1 +%define %%T1 %2 +%define %%CYPH_PLAIN_OUT %3 +%define %%PLAIN_CYPH_IN %4 +%define %%PLAIN_CYPH_LEN %5 +%define %%ENC_DEC %6 +%define %%DATA_OFFSET %7 + + ;; %%PLAIN_CYPH_IN + %%DATA_OFFSET + ;; - input data address + ;; r13 - input data length + ;; rax - temp registers + ;; out: + ;; T1 - packed output + ;; k1 - valid byte mask + READ_SMALL_DATA_INPUT %%T1, %%PLAIN_CYPH_IN+%%DATA_OFFSET, r13, rax + + ;; At this point T1 contains the partial block data + ;; Plaintext XOR E(K, Yn) + vpxorq %%KEY, %%KEY, %%T1 + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; Output r13 Bytes + vmovdqu8 [%%CYPH_PLAIN_OUT + %%DATA_OFFSET]{k1}, %%KEY + +%ifidn %%ENC_DEC, DEC + ;; If decrypt, restore the ciphertext into %%KEY + vmovdqa64 %%KEY, %%T1 +%else + vmovdqu8 %%KEY{k1}{z}, %%KEY +%endif +%endmacro ; ENCRYPT_FINAL_PARTIAL_BLOCK + + + +; Encryption of a single block +%macro ENCRYPT_SINGLE_BLOCK 2 +%define %%GDATA %1 +%define %%XMM0 %2 + + vpxor %%XMM0, %%XMM0, [%%GDATA+16*0] +%assign i 1 +%rep NROUNDS + vaesenc %%XMM0, [%%GDATA+16*i] +%assign i (i+1) +%endrep + vaesenclast %%XMM0, [%%GDATA+16*i] +%endmacro + + +;; Start of Stack Setup + +%macro FUNC_SAVE 0 + ;; Required for Update/GMC_ENC + ;the number of pushes must equal STACK_OFFSET + push r12 + push r13 + push r14 + push r15 + mov r14, rsp + + sub rsp, VARIABLE_OFFSET + and rsp, ~63 + +%ifidn __OUTPUT_FORMAT__, win64 + ; xmm6:xmm15 need to be maintained for Windows + vmovdqu [rsp + LOCAL_STORAGE + 0*16],xmm6 + vmovdqu [rsp + LOCAL_STORAGE + 1*16],xmm7 + vmovdqu [rsp + LOCAL_STORAGE + 2*16],xmm8 + vmovdqu [rsp + LOCAL_STORAGE + 3*16],xmm9 + vmovdqu [rsp + LOCAL_STORAGE + 4*16],xmm10 + vmovdqu [rsp + LOCAL_STORAGE + 5*16],xmm11 + vmovdqu [rsp + LOCAL_STORAGE + 6*16],xmm12 + vmovdqu [rsp + LOCAL_STORAGE + 7*16],xmm13 + vmovdqu [rsp + LOCAL_STORAGE + 8*16],xmm14 + vmovdqu [rsp + LOCAL_STORAGE + 9*16],xmm15 +%endif +%endmacro + + +%macro FUNC_RESTORE 0 + +%ifdef SAFE_DATA + clear_scratch_gps_asm + clear_scratch_zmms_asm +%endif +%ifidn __OUTPUT_FORMAT__, win64 + vmovdqu xmm15, [rsp + LOCAL_STORAGE + 9*16] + vmovdqu xmm14, [rsp + LOCAL_STORAGE + 8*16] + vmovdqu xmm13, [rsp + LOCAL_STORAGE + 7*16] + vmovdqu xmm12, [rsp + LOCAL_STORAGE + 6*16] + vmovdqu xmm11, [rsp + LOCAL_STORAGE + 5*16] + vmovdqu xmm10, [rsp + LOCAL_STORAGE + 4*16] + vmovdqu xmm9, [rsp + LOCAL_STORAGE + 3*16] + vmovdqu xmm8, [rsp + LOCAL_STORAGE + 2*16] + vmovdqu xmm7, [rsp + LOCAL_STORAGE + 1*16] + vmovdqu xmm6, [rsp + LOCAL_STORAGE + 0*16] +%endif +;; Required for Update/GMC_ENC + mov rsp, r14 + pop r15 + pop r14 + pop r13 + pop r12 +%endmacro + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; GCM_INIT initializes a gcm_context_data struct to prepare for encoding/decoding. +; Input: gcm_key_data * (GDATA_KEY), gcm_context_data *(GDATA_CTX), IV, +; Additional Authentication data (A_IN), Additional Data length (A_LEN). +; Output: Updated GDATA_CTX with the hash of A_IN (AadHash) and initialized other parts of GDATA_CTX. +; Clobbers rax, r10-r13, and xmm0-xmm6 +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +%macro GCM_INIT 8 +%define %%GDATA_KEY %1 ; [in] GCM expanded keys pointer +%define %%GDATA_CTX %2 ; [in] GCM context pointer +%define %%IV %3 ; [in] IV pointer +%define %%A_IN %4 ; [in] AAD pointer +%define %%A_LEN %5 ; [in] AAD length in bytes +%define %%GPR1 %6 ; temp GPR +%define %%GPR2 %7 ; temp GPR +%define %%GPR3 %8 ; temp GPR + +%define %%AAD_HASH xmm14 + + CALC_AAD_HASH %%A_IN, %%A_LEN, %%AAD_HASH, %%GDATA_KEY, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, %%GPR1, %%GPR2, %%GPR3 + + mov %%GPR1, %%A_LEN + vmovdqu [%%GDATA_CTX + AadHash], %%AAD_HASH ; ctx_data.aad hash = aad_hash + mov [%%GDATA_CTX + AadLen], %%GPR1 ; ctx_data.aad_length = aad_length + + xor %%GPR1, %%GPR1 + mov [%%GDATA_CTX + InLen], %%GPR1 ; ctx_data.in_length = 0 + mov [%%GDATA_CTX + PBlockLen], %%GPR1 ; ctx_data.partial_block_length = 0 + + ;; read 12 IV bytes and pad with 0x00000001 + mov %%GPR2, %%IV + vmovd xmm3, [%%GPR2 + 8] + vpslldq xmm3, 8 + vmovq xmm2, [%%GPR2] + vmovdqa xmm4, [rel ONEf] + vpternlogq xmm2, xmm3, xmm4, 0xfe ; xmm2 = xmm2 or xmm3 or xmm4 + + vmovdqu [%%GDATA_CTX + OrigIV], xmm2 ; ctx_data.orig_IV = iv + + ;; store IV as counter in LE format + vpshufb xmm2, [rel SHUF_MASK] + vmovdqu [%%GDATA_CTX + CurCount], xmm2 ; ctx_data.current_counter = iv +%endmacro + +%macro GCM_ENC_DEC_SMALL 12 +%define %%GDATA_KEY %1 +%define %%GDATA_CTX %2 +%define %%CYPH_PLAIN_OUT %3 +%define %%PLAIN_CYPH_IN %4 +%define %%PLAIN_CYPH_LEN %5 +%define %%ENC_DEC %6 +%define %%DATA_OFFSET %7 +%define %%LENGTH %8 ; assumed r13 +%define %%NUM_BLOCKS %9 +%define %%CTR %10 ; assumed xmm9 +%define %%HASH_OUT %11 ; assumed xmm14 +%define %%INSTANCE_TYPE %12 + + ;; NOTE: the check below is obsolete in current implementation. The check is already done in GCM_ENC_DEC. + ;; cmp %%NUM_BLOCKS, 0 + ;; je %%_small_initial_blocks_encrypted + cmp %%NUM_BLOCKS, 8 + je %%_small_initial_num_blocks_is_8 + cmp %%NUM_BLOCKS, 7 + je %%_small_initial_num_blocks_is_7 + cmp %%NUM_BLOCKS, 6 + je %%_small_initial_num_blocks_is_6 + cmp %%NUM_BLOCKS, 5 + je %%_small_initial_num_blocks_is_5 + cmp %%NUM_BLOCKS, 4 + je %%_small_initial_num_blocks_is_4 + cmp %%NUM_BLOCKS, 3 + je %%_small_initial_num_blocks_is_3 + cmp %%NUM_BLOCKS, 2 + je %%_small_initial_num_blocks_is_2 + + jmp %%_small_initial_num_blocks_is_1 + + +%%_small_initial_num_blocks_is_8: + ;; r13 - %%LENGTH + ;; xmm12 - T1 + ;; xmm13 - T2 + ;; xmm14 - T3 - AAD HASH OUT when not producing 8 AES keys + ;; xmm15 - T4 + ;; xmm11 - T5 + ;; xmm9 - CTR + ;; xmm1 - XMM1 - Cipher + Hash when producing 8 AES keys + ;; xmm2 - XMM2 + ;; xmm3 - XMM3 + ;; xmm4 - XMM4 + ;; xmm5 - XMM5 + ;; xmm6 - XMM6 + ;; xmm7 - XMM7 + ;; xmm8 - XMM8 - AAD HASH IN + ;; xmm10 - T6 + ;; xmm0 - T_key + INITIAL_BLOCKS_PARTIAL %%GDATA_KEY, %%GDATA_CTX, %%CYPH_PLAIN_OUT, \ + %%PLAIN_CYPH_IN, %%LENGTH, %%DATA_OFFSET, 8, \ + xmm12, xmm13, %%HASH_OUT, xmm15, xmm11, %%CTR, \ + xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm8, \ + xmm10, xmm0, %%ENC_DEC, %%INSTANCE_TYPE + jmp %%_small_initial_blocks_encrypted + +%%_small_initial_num_blocks_is_7: + INITIAL_BLOCKS_PARTIAL %%GDATA_KEY, %%GDATA_CTX, %%CYPH_PLAIN_OUT, \ + %%PLAIN_CYPH_IN, %%LENGTH, %%DATA_OFFSET, 7, \ + xmm12, xmm13, %%HASH_OUT, xmm15, xmm11, %%CTR, \ + xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm8, \ + xmm10, xmm0, %%ENC_DEC, %%INSTANCE_TYPE + jmp %%_small_initial_blocks_encrypted + +%%_small_initial_num_blocks_is_6: + INITIAL_BLOCKS_PARTIAL %%GDATA_KEY, %%GDATA_CTX, %%CYPH_PLAIN_OUT, \ + %%PLAIN_CYPH_IN, %%LENGTH, %%DATA_OFFSET, 6, \ + xmm12, xmm13, %%HASH_OUT, xmm15, xmm11, %%CTR, \ + xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm8, \ + xmm10, xmm0, %%ENC_DEC, %%INSTANCE_TYPE + jmp %%_small_initial_blocks_encrypted + +%%_small_initial_num_blocks_is_5: + INITIAL_BLOCKS_PARTIAL %%GDATA_KEY, %%GDATA_CTX, %%CYPH_PLAIN_OUT, \ + %%PLAIN_CYPH_IN, %%LENGTH, %%DATA_OFFSET, 5, \ + xmm12, xmm13, %%HASH_OUT, xmm15, xmm11, %%CTR, \ + xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm8, \ + xmm10, xmm0, %%ENC_DEC, %%INSTANCE_TYPE + jmp %%_small_initial_blocks_encrypted + +%%_small_initial_num_blocks_is_4: + INITIAL_BLOCKS_PARTIAL %%GDATA_KEY, %%GDATA_CTX, %%CYPH_PLAIN_OUT, \ + %%PLAIN_CYPH_IN, %%LENGTH, %%DATA_OFFSET, 4, \ + xmm12, xmm13, %%HASH_OUT, xmm15, xmm11, %%CTR, \ + xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm8, \ + xmm10, xmm0, %%ENC_DEC, %%INSTANCE_TYPE + jmp %%_small_initial_blocks_encrypted + +%%_small_initial_num_blocks_is_3: + INITIAL_BLOCKS_PARTIAL %%GDATA_KEY, %%GDATA_CTX, %%CYPH_PLAIN_OUT, \ + %%PLAIN_CYPH_IN, %%LENGTH, %%DATA_OFFSET, 3, \ + xmm12, xmm13, %%HASH_OUT, xmm15, xmm11, %%CTR, \ + xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm8, \ + xmm10, xmm0, %%ENC_DEC, %%INSTANCE_TYPE + jmp %%_small_initial_blocks_encrypted + +%%_small_initial_num_blocks_is_2: + INITIAL_BLOCKS_PARTIAL %%GDATA_KEY, %%GDATA_CTX, %%CYPH_PLAIN_OUT, \ + %%PLAIN_CYPH_IN, %%LENGTH, %%DATA_OFFSET, 2, \ + xmm12, xmm13, %%HASH_OUT, xmm15, xmm11, %%CTR, \ + xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm8, \ + xmm10, xmm0, %%ENC_DEC, %%INSTANCE_TYPE + jmp %%_small_initial_blocks_encrypted + +%%_small_initial_num_blocks_is_1: + INITIAL_BLOCKS_PARTIAL %%GDATA_KEY, %%GDATA_CTX, %%CYPH_PLAIN_OUT, \ + %%PLAIN_CYPH_IN, %%LENGTH, %%DATA_OFFSET, 1, \ + xmm12, xmm13, %%HASH_OUT, xmm15, xmm11, %%CTR, \ + xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm8, \ + xmm10, xmm0, %%ENC_DEC, %%INSTANCE_TYPE +%%_small_initial_blocks_encrypted: + +%endmacro ; GCM_ENC_DEC_SMALL + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; GCM_ENC_DEC Encodes/Decodes given data. Assumes that the passed gcm_context_data struct +; has been initialized by GCM_INIT +; Requires the input data be at least 1 byte long because of READ_SMALL_INPUT_DATA. +; Input: gcm_key_data struct* (GDATA_KEY), gcm_context_data *(GDATA_CTX), input text (PLAIN_CYPH_IN), +; input text length (PLAIN_CYPH_LEN) and whether encoding or decoding (ENC_DEC). +; Output: A cypher of the given plain text (CYPH_PLAIN_OUT), and updated GDATA_CTX +; Clobbers rax, r10-r15, and xmm0-xmm15 +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +%macro GCM_ENC_DEC 7 +%define %%GDATA_KEY %1 +%define %%GDATA_CTX %2 +%define %%CYPH_PLAIN_OUT %3 +%define %%PLAIN_CYPH_IN %4 +%define %%PLAIN_CYPH_LEN %5 +%define %%ENC_DEC %6 +%define %%INSTANCE_TYPE %7 +%define %%DATA_OFFSET r11 + +; Macro flow: +; calculate the number of 16byte blocks in the message +; process (number of 16byte blocks) mod 8 '%%_initial_num_blocks_is_# .. %%_initial_blocks_encrypted' +; process 8 16 byte blocks at a time until all are done '%%_encrypt_by_8_new .. %%_eight_cipher_left' +; if there is a block of less tahn 16 bytes process it '%%_zero_cipher_left .. %%_multiple_of_16_bytes' + +%ifidn __OUTPUT_FORMAT__, win64 + cmp %%PLAIN_CYPH_LEN, 0 +%else + or %%PLAIN_CYPH_LEN, %%PLAIN_CYPH_LEN +%endif + je %%_enc_dec_done + + xor %%DATA_OFFSET, %%DATA_OFFSET + ;; Update length of data processed +%ifidn __OUTPUT_FORMAT__, win64 + mov rax, %%PLAIN_CYPH_LEN + add [%%GDATA_CTX + InLen], rax +%else + add [%%GDATA_CTX + InLen], %%PLAIN_CYPH_LEN +%endif + vmovdqu xmm13, [%%GDATA_KEY + HashKey] + vmovdqu xmm8, [%%GDATA_CTX + AadHash] + +%ifidn %%INSTANCE_TYPE, multi_call + ;; NOTE: partial block processing makes only sense for multi_call here. + ;; Used for the update flow - if there was a previous partial + ;; block fill the remaining bytes here. + PARTIAL_BLOCK %%GDATA_KEY, %%GDATA_CTX, %%CYPH_PLAIN_OUT, %%PLAIN_CYPH_IN, %%PLAIN_CYPH_LEN, %%DATA_OFFSET, xmm8, %%ENC_DEC +%endif + + ;; lift CTR set from initial_blocks to here +%ifidn %%INSTANCE_TYPE, single_call + vmovdqu xmm9, xmm2 +%else + vmovdqu xmm9, [%%GDATA_CTX + CurCount] +%endif + + ;; Save the amount of data left to process in r10 + mov r13, %%PLAIN_CYPH_LEN +%ifidn %%INSTANCE_TYPE, multi_call + ;; NOTE: %%DATA_OFFSET is zero in single_call case. + ;; Consequently PLAIN_CYPH_LEN will never be zero after + ;; %%DATA_OFFSET subtraction below. + sub r13, %%DATA_OFFSET + + ;; There may be no more data if it was consumed in the partial block. + cmp r13, 0 + je %%_enc_dec_done +%endif ; %%INSTANCE_TYPE, multi_call + mov r10, r13 + + ;; Determine how many blocks to process in INITIAL + mov r12, r13 + shr r12, 4 + and r12, 7 + + ;; Process one additional block in INITIAL if there is a partial block + and r10, 0xf + blsmsk r10, r10 ; Set CF if zero + cmc ; Flip CF + adc r12, 0x0 ; Process an additional INITIAL block if CF set + + ;; Less than 127B will be handled by the small message code, which + ;; can process up to 7 16B blocks. + cmp r13, 128 + jge %%_large_message_path + + GCM_ENC_DEC_SMALL %%GDATA_KEY, %%GDATA_CTX, %%CYPH_PLAIN_OUT, %%PLAIN_CYPH_IN, %%PLAIN_CYPH_LEN, %%ENC_DEC, %%DATA_OFFSET, r13, r12, xmm9, xmm14, %%INSTANCE_TYPE + jmp %%_ghash_done + +%%_large_message_path: + and r12, 0x7 ; Still, don't allow 8 INITIAL blocks since this will + ; can be handled by the x8 partial loop. + + cmp r12, 0 + je %%_initial_num_blocks_is_0 + cmp r12, 7 + je %%_initial_num_blocks_is_7 + cmp r12, 6 + je %%_initial_num_blocks_is_6 + cmp r12, 5 + je %%_initial_num_blocks_is_5 + cmp r12, 4 + je %%_initial_num_blocks_is_4 + cmp r12, 3 + je %%_initial_num_blocks_is_3 + cmp r12, 2 + je %%_initial_num_blocks_is_2 + + jmp %%_initial_num_blocks_is_1 + +%%_initial_num_blocks_is_7: + ;; r13 - %%LENGTH + ;; xmm12 - T1 + ;; xmm13 - T2 + ;; xmm14 - T3 - AAD HASH OUT when not producing 8 AES keys + ;; xmm15 - T4 + ;; xmm11 - T5 + ;; xmm9 - CTR + ;; xmm1 - XMM1 - Cipher + Hash when producing 8 AES keys + ;; xmm2 - XMM2 + ;; xmm3 - XMM3 + ;; xmm4 - XMM4 + ;; xmm5 - XMM5 + ;; xmm6 - XMM6 + ;; xmm7 - XMM7 + ;; xmm8 - XMM8 - AAD HASH IN + ;; xmm10 - T6 + ;; xmm0 - T_key + INITIAL_BLOCKS %%GDATA_KEY, %%CYPH_PLAIN_OUT, %%PLAIN_CYPH_IN, r13, %%DATA_OFFSET, 7, xmm12, xmm13, xmm14, xmm15, xmm11, xmm9, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm8, xmm10, xmm0, %%ENC_DEC + jmp %%_initial_blocks_encrypted + +%%_initial_num_blocks_is_6: + INITIAL_BLOCKS %%GDATA_KEY, %%CYPH_PLAIN_OUT, %%PLAIN_CYPH_IN, r13, %%DATA_OFFSET, 6, xmm12, xmm13, xmm14, xmm15, xmm11, xmm9, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm8, xmm10, xmm0, %%ENC_DEC + jmp %%_initial_blocks_encrypted + +%%_initial_num_blocks_is_5: + INITIAL_BLOCKS %%GDATA_KEY, %%CYPH_PLAIN_OUT, %%PLAIN_CYPH_IN, r13, %%DATA_OFFSET, 5, xmm12, xmm13, xmm14, xmm15, xmm11, xmm9, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm8, xmm10, xmm0, %%ENC_DEC + jmp %%_initial_blocks_encrypted + +%%_initial_num_blocks_is_4: + INITIAL_BLOCKS %%GDATA_KEY, %%CYPH_PLAIN_OUT, %%PLAIN_CYPH_IN, r13, %%DATA_OFFSET, 4, xmm12, xmm13, xmm14, xmm15, xmm11, xmm9, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm8, xmm10, xmm0, %%ENC_DEC + jmp %%_initial_blocks_encrypted + +%%_initial_num_blocks_is_3: + INITIAL_BLOCKS %%GDATA_KEY, %%CYPH_PLAIN_OUT, %%PLAIN_CYPH_IN, r13, %%DATA_OFFSET, 3, xmm12, xmm13, xmm14, xmm15, xmm11, xmm9, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm8, xmm10, xmm0, %%ENC_DEC + jmp %%_initial_blocks_encrypted + +%%_initial_num_blocks_is_2: + INITIAL_BLOCKS %%GDATA_KEY, %%CYPH_PLAIN_OUT, %%PLAIN_CYPH_IN, r13, %%DATA_OFFSET, 2, xmm12, xmm13, xmm14, xmm15, xmm11, xmm9, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm8, xmm10, xmm0, %%ENC_DEC + jmp %%_initial_blocks_encrypted + +%%_initial_num_blocks_is_1: + INITIAL_BLOCKS %%GDATA_KEY, %%CYPH_PLAIN_OUT, %%PLAIN_CYPH_IN, r13, %%DATA_OFFSET, 1, xmm12, xmm13, xmm14, xmm15, xmm11, xmm9, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm8, xmm10, xmm0, %%ENC_DEC + jmp %%_initial_blocks_encrypted + +%%_initial_num_blocks_is_0: + INITIAL_BLOCKS %%GDATA_KEY, %%CYPH_PLAIN_OUT, %%PLAIN_CYPH_IN, r13, %%DATA_OFFSET, 0, xmm12, xmm13, xmm14, xmm15, xmm11, xmm9, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm8, xmm10, xmm0, %%ENC_DEC + + +%%_initial_blocks_encrypted: + ;; The entire message was encrypted processed in initial and now need to be hashed + cmp r13, 0 + je %%_encrypt_done + + ;; Encrypt the final <16 byte (partial) block, then hash + cmp r13, 16 + jl %%_encrypt_final_partial + + ;; Process 7 full blocks plus a partial block + cmp r13, 128 + jl %%_encrypt_by_8_partial + + +%%_encrypt_by_8_parallel: + ;; in_order vs. out_order is an optimization to increment the counter without shuffling + ;; it back into little endian. r15d keeps track of when we need to increent in order so + ;; that the carry is handled correctly. + vmovd r15d, xmm9 + and r15d, 255 + vpshufb xmm9, [rel SHUF_MASK] + + +%%_encrypt_by_8_new: + cmp r15d, 255-8 + jg %%_encrypt_by_8 + + + + ;; xmm0 - T1 + ;; xmm10 - T2 + ;; xmm11 - T3 + ;; xmm12 - T4 + ;; xmm13 - T5 + ;; xmm14 - T6 + ;; xmm9 - CTR + ;; xmm1 - XMM1 + ;; xmm2 - XMM2 + ;; xmm3 - XMM3 + ;; xmm4 - XMM4 + ;; xmm5 - XMM5 + ;; xmm6 - XMM6 + ;; xmm7 - XMM7 + ;; xmm8 - XMM8 + ;; xmm15 - T7 + add r15b, 8 + GHASH_8_ENCRYPT_8_PARALLEL %%GDATA_KEY, %%CYPH_PLAIN_OUT, %%PLAIN_CYPH_IN, %%DATA_OFFSET, xmm0, xmm10, xmm11, xmm12, xmm13, xmm14, xmm9, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm8, xmm15, out_order, %%ENC_DEC, full + add %%DATA_OFFSET, 128 + sub r13, 128 + cmp r13, 128 + jge %%_encrypt_by_8_new + + vpshufb xmm9, [rel SHUF_MASK] + jmp %%_encrypt_by_8_parallel_done + +%%_encrypt_by_8: + vpshufb xmm9, [rel SHUF_MASK] + add r15b, 8 + GHASH_8_ENCRYPT_8_PARALLEL %%GDATA_KEY, %%CYPH_PLAIN_OUT, %%PLAIN_CYPH_IN, %%DATA_OFFSET, xmm0, xmm10, xmm11, xmm12, xmm13, xmm14, xmm9, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm8, xmm15, in_order, %%ENC_DEC, full + vpshufb xmm9, [rel SHUF_MASK] + add %%DATA_OFFSET, 128 + sub r13, 128 + cmp r13, 128 + jge %%_encrypt_by_8_new + vpshufb xmm9, [rel SHUF_MASK] + + +%%_encrypt_by_8_parallel_done: + ;; Test to see if we need a by 8 with partial block. At this point + ;; bytes remaining should be either zero or between 113-127. + cmp r13, 0 + je %%_encrypt_done + +%%_encrypt_by_8_partial: + ;; Shuffle needed to align key for partial block xor. out_order + ;; is a little faster because it avoids extra shuffles. + ;; TBD: Might need to account for when we don't have room to increment the counter. + + + ;; Process parallel buffers with a final partial block. + GHASH_8_ENCRYPT_8_PARALLEL %%GDATA_KEY, %%CYPH_PLAIN_OUT, %%PLAIN_CYPH_IN, %%DATA_OFFSET, xmm0, xmm10, xmm11, xmm12, xmm13, xmm14, xmm9, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm8, xmm15, in_order, %%ENC_DEC, partial + + + add %%DATA_OFFSET, 128-16 + sub r13, 128-16 + +%%_encrypt_final_partial: + + vpshufb xmm8, [rel SHUF_MASK] + mov [%%GDATA_CTX + PBlockLen], r13 + vmovdqu [%%GDATA_CTX + PBlockEncKey], xmm8 + + ;; xmm8 - Final encrypted counter - need to hash with partial or full block ciphertext + ;; GDATA, KEY, T1, T2 + ENCRYPT_FINAL_PARTIAL_BLOCK xmm8, xmm0, %%CYPH_PLAIN_OUT, %%PLAIN_CYPH_IN, %%PLAIN_CYPH_LEN, %%ENC_DEC, %%DATA_OFFSET + + vpshufb xmm8, [rel SHUF_MASK] + + +%%_encrypt_done: + + ;; Mapping to macro parameters + ;; IN: + ;; xmm9 contains the counter + ;; xmm1-xmm8 contain the xor'd ciphertext + ;; OUT: + ;; xmm14 contains the final hash + ;; GDATA, T1, T2, T3, T4, T5, T6, T7, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm8 +%ifidn %%INSTANCE_TYPE, multi_call + mov r13, [%%GDATA_CTX + PBlockLen] + cmp r13, 0 + jz %%_hash_last_8 + GHASH_LAST_7 %%GDATA_KEY, xmm0, xmm10, xmm11, xmm12, xmm13, xmm14, xmm15, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7 + ;; XOR the partial word into the hash + vpxor xmm14, xmm14, xmm8 + jmp %%_ghash_done +%endif +%%_hash_last_8: + GHASH_LAST_8 %%GDATA_KEY, xmm0, xmm10, xmm11, xmm12, xmm13, xmm14, xmm15, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm8 + +%%_ghash_done: + vmovdqu [%%GDATA_CTX + CurCount], xmm9 ; my_ctx_data.current_counter = xmm9 + vmovdqu [%%GDATA_CTX + AadHash], xmm14 ; my_ctx_data.aad hash = xmm14 + +%%_enc_dec_done: + + +%endmacro ; GCM_ENC_DEC + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; GCM_COMPLETE Finishes Encyrption/Decryption of last partial block after GCM_UPDATE finishes. +; Input: A gcm_key_data * (GDATA_KEY), gcm_context_data (GDATA_CTX) and whether encoding or decoding (ENC_DEC). +; Output: Authorization Tag (AUTH_TAG) and Authorization Tag length (AUTH_TAG_LEN) +; Clobbers rax, r10-r12, and xmm0, xmm1, xmm5, xmm6, xmm9, xmm11, xmm14, xmm15 +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +%macro GCM_COMPLETE 6 +%define %%GDATA_KEY %1 +%define %%GDATA_CTX %2 +%define %%AUTH_TAG %3 +%define %%AUTH_TAG_LEN %4 +%define %%ENC_DEC %5 +%define %%INSTANCE_TYPE %6 +%define %%PLAIN_CYPH_LEN rax + + vmovdqu xmm13, [%%GDATA_KEY + HashKey] + ;; Start AES as early as possible + vmovdqu xmm9, [%%GDATA_CTX + OrigIV] ; xmm9 = Y0 + ENCRYPT_SINGLE_BLOCK %%GDATA_KEY, xmm9 ; E(K, Y0) + +%ifidn %%INSTANCE_TYPE, multi_call + ;; If the GCM function is called as a single function call rather + ;; than invoking the individual parts (init, update, finalize) we + ;; can remove a write to read dependency on AadHash. + vmovdqu xmm14, [%%GDATA_CTX + AadHash] + + ;; Encrypt the final partial block. If we did this as a single call then + ;; the partial block was handled in the main GCM_ENC_DEC macro. + mov r12, [%%GDATA_CTX + PBlockLen] + cmp r12, 0 + + je %%_partial_done + + GHASH_MUL xmm14, xmm13, xmm0, xmm10, xmm11, xmm5, xmm6 ;GHASH computation for the last <16 Byte block + vmovdqu [%%GDATA_CTX + AadHash], xmm14 + +%%_partial_done: + +%endif + + mov r12, [%%GDATA_CTX + AadLen] ; r12 = aadLen (number of bytes) + mov %%PLAIN_CYPH_LEN, [%%GDATA_CTX + InLen] + + shl r12, 3 ; convert into number of bits + vmovd xmm15, r12d ; len(A) in xmm15 + + shl %%PLAIN_CYPH_LEN, 3 ; len(C) in bits (*128) + vmovq xmm1, %%PLAIN_CYPH_LEN + vpslldq xmm15, xmm15, 8 ; xmm15 = len(A)|| 0x0000000000000000 + vpxor xmm15, xmm15, xmm1 ; xmm15 = len(A)||len(C) + + vpxor xmm14, xmm15 + GHASH_MUL xmm14, xmm13, xmm0, xmm10, xmm11, xmm5, xmm6 + vpshufb xmm14, [rel SHUF_MASK] ; perform a 16Byte swap + + vpxor xmm9, xmm9, xmm14 + + +%%_return_T: + mov r10, %%AUTH_TAG ; r10 = authTag + mov r11, %%AUTH_TAG_LEN ; r11 = auth_tag_len + + cmp r11, 16 + je %%_T_16 + + cmp r11, 12 + je %%_T_12 + + cmp r11, 8 + je %%_T_8 + + simd_store_avx r10, xmm9, r11, r12, rax + jmp %%_return_T_done +%%_T_8: + vmovq rax, xmm9 + mov [r10], rax + jmp %%_return_T_done +%%_T_12: + vmovq rax, xmm9 + mov [r10], rax + vpsrldq xmm9, xmm9, 8 + vmovd eax, xmm9 + mov [r10 + 8], eax + jmp %%_return_T_done +%%_T_16: + vmovdqu [r10], xmm9 + +%%_return_T_done: + +%ifdef SAFE_DATA + ;; Clear sensitive data from context structure + vpxor xmm0, xmm0 + vmovdqu [%%GDATA_CTX + AadHash], xmm0 + vmovdqu [%%GDATA_CTX + PBlockEncKey], xmm0 +%endif +%endmacro ; GCM_COMPLETE + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;void aes_gcm_precomp_128_avx512 / +; aes_gcm_precomp_192_avx512 / +; aes_gcm_precomp_256_avx512 +; (struct gcm_key_data *key_data) +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +MKGLOBAL(FN_NAME(precomp,_),function,) +FN_NAME(precomp,_): +;; Parameter is passed through register +%ifdef SAFE_PARAM + ;; Check key_data != NULL + cmp arg1, 0 + jz exit_precomp +%endif + + push r12 + push r13 + push r14 + push r15 + + mov r14, rsp + + + + sub rsp, VARIABLE_OFFSET + and rsp, ~63 ; align rsp to 64 bytes + +%ifidn __OUTPUT_FORMAT__, win64 + ; only xmm6 needs to be maintained + vmovdqu [rsp + LOCAL_STORAGE + 0*16],xmm6 +%endif + + vpxor xmm6, xmm6 + ENCRYPT_SINGLE_BLOCK arg1, xmm6 ; xmm6 = HashKey + + vpshufb xmm6, [rel SHUF_MASK] + ;;;;;;;;;;;;;;; PRECOMPUTATION of HashKey<<1 mod poly from the HashKey;;;;;;;;;;;;;;; + vmovdqa xmm2, xmm6 + vpsllq xmm6, xmm6, 1 + vpsrlq xmm2, xmm2, 63 + vmovdqa xmm1, xmm2 + vpslldq xmm2, xmm2, 8 + vpsrldq xmm1, xmm1, 8 + vpor xmm6, xmm6, xmm2 + ;reduction + vpshufd xmm2, xmm1, 00100100b + vpcmpeqd xmm2, [rel TWOONE] + vpand xmm2, xmm2, [rel POLY] + vpxor xmm6, xmm6, xmm2 ; xmm6 holds the HashKey<<1 mod poly + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + vmovdqu [arg1 + HashKey], xmm6 ; store HashKey<<1 mod poly + + + PRECOMPUTE arg1, xmm6, xmm0, xmm1, xmm2, xmm3, xmm4, xmm5 + +%ifdef SAFE_DATA + clear_scratch_gps_asm + clear_scratch_zmms_asm +%endif +%ifidn __OUTPUT_FORMAT__, win64 + vmovdqu xmm6, [rsp + LOCAL_STORAGE + 0*16] +%endif + mov rsp, r14 + + pop r15 + pop r14 + pop r13 + pop r12 + +exit_precomp: + ret + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;void aes_gcm_init_128_avx512 / aes_gcm_init_192_avx512 / aes_gcm_init_256_avx512 +; (const struct gcm_key_data *key_data, +; struct gcm_context_data *context_data, +; u8 *iv, +; const u8 *aad, +; u64 aad_len); +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +MKGLOBAL(FN_NAME(init,_),function,) +FN_NAME(init,_): + push r12 + push r13 +%ifidn __OUTPUT_FORMAT__, win64 + push r14 + push r15 + mov r14, rsp + ; xmm6:xmm15 need to be maintained for Windows + sub rsp, 1*16 + movdqu [rsp + 0*16], xmm6 +%endif + +%ifdef SAFE_PARAM + ;; Check key_data != NULL + cmp arg1, 0 + jz exit_init + + ;; Check context_data != NULL + cmp arg2, 0 + jz exit_init + + ;; Check IV != NULL + cmp arg3, 0 + jz exit_init + + ;; Check if aad_len == 0 + cmp arg5, 0 + jz skip_aad_check_init + + ;; Check aad != NULL (aad_len != 0) + cmp arg4, 0 + jz exit_init + +skip_aad_check_init: +%endif + GCM_INIT arg1, arg2, arg3, arg4, arg5, r10, r11, r12 + +%ifdef SAFE_DATA + clear_scratch_gps_asm + clear_scratch_zmms_asm +%endif +exit_init: +%ifidn __OUTPUT_FORMAT__, win64 + movdqu xmm6 , [rsp + 0*16] + mov rsp, r14 + pop r15 + pop r14 +%endif + pop r13 + pop r12 + ret + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;void aes_gcm_enc_128_update_avx512 / aes_gcm_enc_192_update_avx512 / +; aes_gcm_enc_256_update_avx512 +; (const struct gcm_key_data *key_data, +; struct gcm_context_data *context_data, +; u8 *out, +; const u8 *in, +; u64 plaintext_len); +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +MKGLOBAL(FN_NAME(enc,_update_),function,) +FN_NAME(enc,_update_): + + FUNC_SAVE + +%ifdef SAFE_PARAM + ;; Check key_data != NULL + cmp arg1, 0 + jz exit_update_enc + + ;; Check context_data != NULL + cmp arg2, 0 + jz exit_update_enc + + ;; Check if plaintext_len == 0 + cmp arg5, 0 + jz skip_in_out_check_update_enc + + ;; Check out != NULL (plaintext_len != 0) + cmp arg3, 0 + jz exit_update_enc + + ;; Check in != NULL (plaintext_len != 0) + cmp arg4, 0 + jz exit_update_enc + +skip_in_out_check_update_enc: +%endif + GCM_ENC_DEC arg1, arg2, arg3, arg4, arg5, ENC, multi_call + +exit_update_enc: + FUNC_RESTORE + + ret + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;void aes_gcm_dec_128_update_avx512 / aes_gcm_dec_192_update_avx512 / +; aes_gcm_dec_256_update_avx512 +; (const struct gcm_key_data *key_data, +; struct gcm_context_data *context_data, +; u8 *out, +; const u8 *in, +; u64 plaintext_len); +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +MKGLOBAL(FN_NAME(dec,_update_),function,) +FN_NAME(dec,_update_): + + FUNC_SAVE + +%ifdef SAFE_PARAM + ;; Check key_data != NULL + cmp arg1, 0 + jz exit_update_dec + + ;; Check context_data != NULL + cmp arg2, 0 + jz exit_update_dec + + ;; Check if plaintext_len == 0 + cmp arg5, 0 + jz skip_in_out_check_update_dec + + ;; Check out != NULL (plaintext_len != 0) + cmp arg3, 0 + jz exit_update_dec + + ;; Check in != NULL (plaintext_len != 0) + cmp arg4, 0 + jz exit_update_dec + +skip_in_out_check_update_dec: +%endif + + GCM_ENC_DEC arg1, arg2, arg3, arg4, arg5, DEC, multi_call + +exit_update_dec: + FUNC_RESTORE + ret + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;void aes_gcm_enc_128_finalize_avx512 / aes_gcm_enc_192_finalize_avx512 / +; aes_gcm_enc_256_finalize_avx512 +; (const struct gcm_key_data *key_data, +; struct gcm_context_data *context_data, +; u8 *auth_tag, +; u64 auth_tag_len); +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +MKGLOBAL(FN_NAME(enc,_finalize_),function,) +FN_NAME(enc,_finalize_): + +;; All parameters are passed through registers +%ifdef SAFE_PARAM + ;; Check key_data != NULL + cmp arg1, 0 + jz exit_enc_fin + + ;; Check context_data != NULL + cmp arg2, 0 + jz exit_enc_fin + + ;; Check auth_tag != NULL + cmp arg3, 0 + jz exit_enc_fin + + ;; Check auth_tag_len == 0 or > 16 + cmp arg4, 0 + jz exit_enc_fin + + cmp arg4, 16 + ja exit_enc_fin +%endif + + push r12 + +%ifidn __OUTPUT_FORMAT__, win64 + ; xmm6:xmm15 need to be maintained for Windows + sub rsp, 5*16 + vmovdqu [rsp + 0*16], xmm6 + vmovdqu [rsp + 1*16], xmm9 + vmovdqu [rsp + 2*16], xmm11 + vmovdqu [rsp + 3*16], xmm14 + vmovdqu [rsp + 4*16], xmm15 +%endif + GCM_COMPLETE arg1, arg2, arg3, arg4, ENC, multi_call + +%ifdef SAFE_DATA + clear_scratch_gps_asm + clear_scratch_zmms_asm +%endif +%ifidn __OUTPUT_FORMAT__, win64 + vmovdqu xmm15, [rsp + 4*16] + vmovdqu xmm14, [rsp + 3*16] + vmovdqu xmm11, [rsp + 2*16] + vmovdqu xmm9, [rsp + 1*16] + vmovdqu xmm6, [rsp + 0*16] + add rsp, 5*16 +%endif + + pop r12 + +exit_enc_fin: + ret + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;void aes_gcm_dec_128_finalize_avx512 / aes_gcm_dec_192_finalize_avx512 +; aes_gcm_dec_256_finalize_avx512 +; (const struct gcm_key_data *key_data, +; struct gcm_context_data *context_data, +; u8 *auth_tag, +; u64 auth_tag_len); +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +MKGLOBAL(FN_NAME(dec,_finalize_),function,) +FN_NAME(dec,_finalize_): + +;; All parameters are passed through registers +%ifdef SAFE_PARAM + ;; Check key_data != NULL + cmp arg1, 0 + jz exit_dec_fin + + ;; Check context_data != NULL + cmp arg2, 0 + jz exit_dec_fin + + ;; Check auth_tag != NULL + cmp arg3, 0 + jz exit_dec_fin + + ;; Check auth_tag_len == 0 or > 16 + cmp arg4, 0 + jz exit_dec_fin + + cmp arg4, 16 + ja exit_dec_fin +%endif + + push r12 + +%ifidn __OUTPUT_FORMAT__, win64 + ; xmm6:xmm15 need to be maintained for Windows + sub rsp, 5*16 + vmovdqu [rsp + 0*16], xmm6 + vmovdqu [rsp + 1*16], xmm9 + vmovdqu [rsp + 2*16], xmm11 + vmovdqu [rsp + 3*16], xmm14 + vmovdqu [rsp + 4*16], xmm15 +%endif + GCM_COMPLETE arg1, arg2, arg3, arg4, DEC, multi_call + +%ifdef SAFE_DATA + clear_scratch_gps_asm + clear_scratch_zmms_asm +%endif +%ifidn __OUTPUT_FORMAT__, win64 + vmovdqu xmm15, [rsp + 4*16] + vmovdqu xmm14, [rsp + 3*16] + vmovdqu xmm11, [rsp + 2*16] + vmovdqu xmm9, [rsp + 1*16] + vmovdqu xmm6, [rsp + 0*16] + add rsp, 5*16 +%endif + + pop r12 +exit_dec_fin: + + ret + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;void aes_gcm_enc_128_avx512 / aes_gcm_enc_192_avx512 / aes_gcm_enc_256_avx512 +; (const struct gcm_key_data *key_data, +; struct gcm_context_data *context_data, +; u8 *out, +; const u8 *in, +; u64 plaintext_len, +; u8 *iv, +; const u8 *aad, +; u64 aad_len, +; u8 *auth_tag, +; u64 auth_tag_len); +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +MKGLOBAL(FN_NAME(enc,_),function,) +FN_NAME(enc,_): + + FUNC_SAVE + +%ifdef SAFE_PARAM + ;; Check key_data != NULL + cmp arg1, 0 + jz exit_enc + + ;; Check context_data != NULL + cmp arg2, 0 + jz exit_enc + + ;; Check IV != NULL + cmp arg6, 0 + jz exit_enc + + ;; Check auth_tag != NULL + cmp arg9, 0 + jz exit_enc + + ;; Check auth_tag_len == 0 or > 16 + cmp arg10, 0 + jz exit_enc + + cmp arg10, 16 + ja exit_enc + + ;; Check if plaintext_len == 0 + cmp arg5, 0 + jz skip_in_out_check_enc + + ;; Check out != NULL (plaintext_len != 0) + cmp arg3, 0 + jz exit_enc + + ;; Check in != NULL (plaintext_len != 0) + cmp arg4, 0 + jz exit_enc + +skip_in_out_check_enc: + ;; Check if aad_len == 0 + cmp arg8, 0 + jz skip_aad_check_enc + + ;; Check aad != NULL (aad_len != 0) + cmp arg7, 0 + jz exit_enc + +skip_aad_check_enc: +%endif + GCM_INIT arg1, arg2, arg6, arg7, arg8, r10, r11, r12 + + GCM_ENC_DEC arg1, arg2, arg3, arg4, arg5, ENC, single_call + + GCM_COMPLETE arg1, arg2, arg9, arg10, ENC, single_call + +exit_enc: + FUNC_RESTORE + + ret + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;void aes_gcm_dec_128_avx512 / aes_gcm_dec_192_avx512 / aes_gcm_dec_256_avx512 +; (const struct gcm_key_data *key_data, +; struct gcm_context_data *context_data, +; u8 *out, +; const u8 *in, +; u64 plaintext_len, +; u8 *iv, +; const u8 *aad, +; u64 aad_len, +; u8 *auth_tag, +; u64 auth_tag_len); +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +MKGLOBAL(FN_NAME(dec,_),function,) +FN_NAME(dec,_): + + FUNC_SAVE + +%ifdef SAFE_PARAM + ;; Check key_data != NULL + cmp arg1, 0 + jz exit_dec + + ;; Check context_data != NULL + cmp arg2, 0 + jz exit_dec + + ;; Check IV != NULL + cmp arg6, 0 + jz exit_dec + + ;; Check auth_tag != NULL + cmp arg9, 0 + jz exit_dec + + ;; Check auth_tag_len == 0 or > 16 + cmp arg10, 0 + jz exit_dec + + cmp arg10, 16 + ja exit_dec + + ;; Check if plaintext_len == 0 + cmp arg5, 0 + jz skip_in_out_check_dec + + ;; Check out != NULL (plaintext_len != 0) + cmp arg3, 0 + jz exit_dec + + ;; Check in != NULL (plaintext_len != 0) + cmp arg4, 0 + jz exit_dec + +skip_in_out_check_dec: + ;; Check if aad_len == 0 + cmp arg8, 0 + jz skip_aad_check_dec + + ;; Check aad != NULL (aad_len != 0) + cmp arg7, 0 + jz exit_dec + +skip_aad_check_dec: +%endif + + GCM_INIT arg1, arg2, arg6, arg7, arg8, r10, r11, r12 + + GCM_ENC_DEC arg1, arg2, arg3, arg4, arg5, DEC, single_call + + GCM_COMPLETE arg1, arg2, arg9, arg10, DEC, single_call + +exit_dec: + FUNC_RESTORE + + ret + +%ifdef LINUX +section .note.GNU-stack noalloc noexec nowrite progbits +%endif diff --git a/src/spdk/intel-ipsec-mb/avx512/gcm_vaes_avx512.asm b/src/spdk/intel-ipsec-mb/avx512/gcm_vaes_avx512.asm new file mode 100644 index 000000000..4ef183d31 --- /dev/null +++ b/src/spdk/intel-ipsec-mb/avx512/gcm_vaes_avx512.asm @@ -0,0 +1,4272 @@ +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; Copyright(c) 2018-2019, Intel Corporation All rights reserved. +; +; Redistribution and use in source and binary forms, with or without +; modification, are permitted provided that the following conditions +; are met: +; * Redistributions of source code must retain the above copyright +; notice, this list of conditions and the following disclaimer. +; * Redistributions in binary form must reproduce the above copyright +; notice, this list of conditions and the following disclaimer in +; the documentation and/or other materials provided with the +; distribution. +; * Neither the name of Intel Corporation nor the names of its +; contributors may be used to endorse or promote products derived +; from this software without specific prior written permission. +; +; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +; "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +; LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +; A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +; OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +; SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +; LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +; DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +; THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +; +; Authors: +; Erdinc Ozturk +; Vinodh Gopal +; James Guilford +; Tomasz Kantecki +; +; +; References: +; This code was derived and highly optimized from the code described in paper: +; Vinodh Gopal et. al. Optimized Galois-Counter-Mode Implementation on Intel Architecture Processors. August, 2010 +; The details of the implementation is explained in: +; Erdinc Ozturk et. al. Enabling High-Performance Galois-Counter-Mode on Intel Architecture Processors. October, 2012. +; +; +; +; +; Assumptions: +; +; +; +; iv: +; 0 1 2 3 +; 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 +; +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +; | Salt (From the SA) | +; +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +; | Initialization Vector | +; | (This is the sequence number from IPSec header) | +; +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +; | 0x1 | +; +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +; +; +; +; AAD: +; AAD will be padded with 0 to the next 16byte multiple +; for example, assume AAD is a u32 vector +; +; if AAD is 8 bytes: +; AAD[3] = {A0, A1}; +; padded AAD in xmm register = {A1 A0 0 0} +; +; 0 1 2 3 +; 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 +; +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +; | SPI (A1) | +; +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +; | 32-bit Sequence Number (A0) | +; +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +; | 0x0 | +; +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +; +; AAD Format with 32-bit Sequence Number +; +; if AAD is 12 bytes: +; AAD[3] = {A0, A1, A2}; +; padded AAD in xmm register = {A2 A1 A0 0} +; +; 0 1 2 3 +; 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 +; +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +; | SPI (A2) | +; +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +; | 64-bit Extended Sequence Number {A1,A0} | +; | | +; +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +; | 0x0 | +; +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +; +; AAD Format with 64-bit Extended Sequence Number +; +; +; aadLen: +; Must be a multiple of 4 bytes and from the definition of the spec. +; The code additionally supports any aadLen length. +; +; TLen: +; from the definition of the spec, TLen can only be 8, 12 or 16 bytes. +; +; poly = x^128 + x^127 + x^126 + x^121 + 1 +; throughout the code, one tab and two tab indentations are used. one tab is for GHASH part, two tabs is for AES part. +; + +%include "include/os.asm" +%include "include/reg_sizes.asm" +%include "include/clear_regs.asm" +%include "include/gcm_defines.asm" +%include "include/gcm_keys_vaes_avx512.asm" +%include "include/memcpy.asm" +%include "include/aes_common.asm" + +%ifndef GCM128_MODE +%ifndef GCM192_MODE +%ifndef GCM256_MODE +%error "No GCM mode selected for gcm_avx512.asm!" +%endif +%endif +%endif + +;; Decide on AES-GCM key size to compile for +%ifdef GCM128_MODE +%define NROUNDS 9 +%define FN_NAME(x,y) aes_gcm_ %+ x %+ _128 %+ y %+ vaes_avx512 +%endif + +%ifdef GCM192_MODE +%define NROUNDS 11 +%define FN_NAME(x,y) aes_gcm_ %+ x %+ _192 %+ y %+ vaes_avx512 +%endif + +%ifdef GCM256_MODE +%define NROUNDS 13 +%define FN_NAME(x,y) aes_gcm_ %+ x %+ _256 %+ y %+ vaes_avx512 +%endif + +section .text +default rel + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;; Stack frame definition +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +%ifidn __OUTPUT_FORMAT__, win64 + %define XMM_STORAGE (10*16) ; space for 10 XMM registers + %define GP_STORAGE ((9*8) + 24) ; space for 9 GP registers + 24 bytes for 64 byte alignment +%else + %define XMM_STORAGE 0 + %define GP_STORAGE (8*8) ; space for 7 GP registers + 1 for alignment +%endif +%ifdef GCM_BIG_DATA +%define LOCAL_STORAGE (128*16) ; space for up to 128 AES blocks +%else +%define LOCAL_STORAGE (48*16) ; space for up to 48 AES blocks +%endif + +;;; sequence is (bottom-up): GP, XMM, local +%define STACK_GP_OFFSET 0 +%define STACK_XMM_OFFSET (STACK_GP_OFFSET + GP_STORAGE) +%define STACK_LOCAL_OFFSET (STACK_XMM_OFFSET + XMM_STORAGE) +%define STACK_FRAME_SIZE (STACK_LOCAL_OFFSET + LOCAL_STORAGE) + +;; for compatibility with stack argument definitions in gcm_defines.asm +%define STACK_OFFSET 0 + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;; Utility Macros +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +;;; =========================================================================== +;;; =========================================================================== +;;; Horizontal XOR - 4 x 128bits xored together +%macro VHPXORI4x128 2 +%define %%REG %1 ; [in/out] ZMM with 4x128bits to xor; 128bit output +%define %%TMP %2 ; [clobbered] ZMM temporary register + vextracti64x4 YWORD(%%TMP), %%REG, 1 + vpxorq YWORD(%%REG), YWORD(%%REG), YWORD(%%TMP) + vextracti32x4 XWORD(%%TMP), YWORD(%%REG), 1 + vpxorq XWORD(%%REG), XWORD(%%REG), XWORD(%%TMP) +%endmacro ; VHPXORI4x128 + +;;; =========================================================================== +;;; =========================================================================== +;;; Horizontal XOR - 2 x 128bits xored together +%macro VHPXORI2x128 2 +%define %%REG %1 ; [in/out] YMM/ZMM with 2x128bits to xor; 128bit output +%define %%TMP %2 ; [clobbered] XMM/YMM/ZMM temporary register + vextracti32x4 XWORD(%%TMP), %%REG, 1 + vpxorq XWORD(%%REG), XWORD(%%REG), XWORD(%%TMP) +%endmacro ; VHPXORI2x128 + +;;; =========================================================================== +;;; =========================================================================== +;;; schoolbook multiply - 1st step +%macro VCLMUL_STEP1 6-7 +%define %%KP %1 ; [in] key pointer +%define %%HI %2 ; [in] previous blocks 4 to 7 +%define %%TMP %3 ; [clobbered] ZMM/YMM/XMM temporary +%define %%TH %4 ; [out] high product +%define %%TM %5 ; [out] medium product +%define %%TL %6 ; [out] low product +%define %%HKEY %7 ; [in/optional] hash key for multiplication + +%if %0 == 6 + vmovdqu64 %%TMP, [%%KP + HashKey_4] +%else + vmovdqa64 %%TMP, %%HKEY +%endif + vpclmulqdq %%TH, %%HI, %%TMP, 0x11 ; %%T5 = a1*b1 + vpclmulqdq %%TL, %%HI, %%TMP, 0x00 ; %%T7 = a0*b0 + vpclmulqdq %%TM, %%HI, %%TMP, 0x01 ; %%T6 = a1*b0 + vpclmulqdq %%TMP, %%HI, %%TMP, 0x10 ; %%T4 = a0*b1 + vpxorq %%TM, %%TM, %%TMP ; [%%TH : %%TM : %%TL] +%endmacro ; VCLMUL_STEP1 + +;;; =========================================================================== +;;; =========================================================================== +;;; schoolbook multiply - 2nd step +%macro VCLMUL_STEP2 9-11 +%define %%KP %1 ; [in] key pointer +%define %%HI %2 ; [out] ghash high 128 bits +%define %%LO %3 ; [in/out] cipher text blocks 0-3 (in); ghash low 128 bits (out) +%define %%TMP0 %4 ; [clobbered] ZMM/YMM/XMM temporary +%define %%TMP1 %5 ; [clobbered] ZMM/YMM/XMM temporary +%define %%TMP2 %6 ; [clobbered] ZMM/YMM/XMM temporary +%define %%TH %7 ; [in] high product +%define %%TM %8 ; [in] medium product +%define %%TL %9 ; [in] low product +%define %%HKEY %10 ; [in/optional] hash key for multiplication +%define %%HXOR %11 ; [in/optional] type of horizontal xor (4 - 4x128; 2 - 2x128; 1 - none) + +%if %0 == 9 + vmovdqu64 %%TMP0, [%%KP + HashKey_8] +%else + vmovdqa64 %%TMP0, %%HKEY +%endif + vpclmulqdq %%TMP1, %%LO, %%TMP0, 0x10 ; %%TMP1 = a0*b1 + vpclmulqdq %%TMP2, %%LO, %%TMP0, 0x11 ; %%TMP2 = a1*b1 + vpxorq %%TH, %%TH, %%TMP2 + vpclmulqdq %%TMP2, %%LO, %%TMP0, 0x00 ; %%TMP2 = a0*b0 + vpxorq %%TL, %%TL, %%TMP2 + vpclmulqdq %%TMP0, %%LO, %%TMP0, 0x01 ; %%TMP0 = a1*b0 + vpternlogq %%TM, %%TMP1, %%TMP0, 0x96 ; %%TM = TM xor TMP1 xor TMP0 + + ;; finish multiplications + vpsrldq %%TMP2, %%TM, 8 + vpxorq %%HI, %%TH, %%TMP2 + vpslldq %%TMP2, %%TM, 8 + vpxorq %%LO, %%TL, %%TMP2 + + ;; xor 128bit words horizontally and compute [(X8*H1) + (X7*H2) + ... ((X1+Y0)*H8] + ;; note: (X1+Y0) handled elsewhere +%if %0 < 11 + VHPXORI4x128 %%HI, %%TMP2 + VHPXORI4x128 %%LO, %%TMP1 +%else +%if %%HXOR == 4 + VHPXORI4x128 %%HI, %%TMP2 + VHPXORI4x128 %%LO, %%TMP1 +%elif %%HXOR == 2 + VHPXORI2x128 %%HI, %%TMP2 + VHPXORI2x128 %%LO, %%TMP1 +%endif ; HXOR + ;; for HXOR == 1 there is nothing to be done +%endif ; !(%0 < 11) + ;; HIx holds top 128 bits + ;; LOx holds low 128 bits + ;; - further reductions to follow +%endmacro ; VCLMUL_STEP2 + +;;; =========================================================================== +;;; =========================================================================== +;;; AVX512 reduction macro +%macro VCLMUL_REDUCE 6 +%define %%OUT %1 ; [out] zmm/ymm/xmm: result (must not be %%TMP1 or %%HI128) +%define %%POLY %2 ; [in] zmm/ymm/xmm: polynomial +%define %%HI128 %3 ; [in] zmm/ymm/xmm: high 128b of hash to reduce +%define %%LO128 %4 ; [in] zmm/ymm/xmm: low 128b of hash to reduce +%define %%TMP0 %5 ; [in] zmm/ymm/xmm: temporary register +%define %%TMP1 %6 ; [in] zmm/ymm/xmm: temporary register + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; first phase of the reduction + vpclmulqdq %%TMP0, %%POLY, %%LO128, 0x01 + vpslldq %%TMP0, %%TMP0, 8 ; shift-L 2 DWs + vpxorq %%TMP0, %%LO128, %%TMP0 ; first phase of the reduction complete + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; second phase of the reduction + vpclmulqdq %%TMP1, %%POLY, %%TMP0, 0x00 + vpsrldq %%TMP1, %%TMP1, 4 ; shift-R only 1-DW to obtain 2-DWs shift-R + + vpclmulqdq %%OUT, %%POLY, %%TMP0, 0x10 + vpslldq %%OUT, %%OUT, 4 ; shift-L 1-DW to obtain result with no shifts + + vpternlogq %%OUT, %%TMP1, %%HI128, 0x96 ; OUT/GHASH = OUT xor TMP1 xor HI128 + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +%endmacro + +;;; =========================================================================== +;;; =========================================================================== +;;; schoolbook multiply (1 to 8 blocks) - 1st step +%macro VCLMUL_1_TO_8_STEP1 8 +%define %%KP %1 ; [in] key pointer +%define %%HI %2 ; [in] ZMM ciphered blocks 4 to 7 +%define %%TMP1 %3 ; [clobbered] ZMM temporary +%define %%TMP2 %4 ; [clobbered] ZMM temporary +%define %%TH %5 ; [out] ZMM high product +%define %%TM %6 ; [out] ZMM medium product +%define %%TL %7 ; [out] ZMM low product +%define %%NBLOCKS %8 ; [in] number of blocks to ghash (0 to 8) + +%if %%NBLOCKS == 8 + VCLMUL_STEP1 %%KP, %%HI, %%TMP1, %%TH, %%TM, %%TL +%elif %%NBLOCKS == 7 + vmovdqu64 %%TMP2, [%%KP + HashKey_3] + vmovdqa64 %%TMP1, [rel mask_out_top_block] + vpandq %%TMP2, %%TMP1 + vpandq %%HI, %%TMP1 + VCLMUL_STEP1 NULL, %%HI, %%TMP1, %%TH, %%TM, %%TL, %%TMP2 +%elif %%NBLOCKS == 6 + vmovdqu64 YWORD(%%TMP2), [%%KP + HashKey_2] + VCLMUL_STEP1 NULL, YWORD(%%HI), YWORD(%%TMP1), \ + YWORD(%%TH), YWORD(%%TM), YWORD(%%TL), YWORD(%%TMP2) +%elif %%NBLOCKS == 5 + vmovdqu64 XWORD(%%TMP2), [%%KP + HashKey_1] + VCLMUL_STEP1 NULL, XWORD(%%HI), XWORD(%%TMP1), \ + XWORD(%%TH), XWORD(%%TM), XWORD(%%TL), XWORD(%%TMP2) +%else + vpxorq %%TH, %%TH + vpxorq %%TM, %%TM + vpxorq %%TL, %%TL +%endif +%endmacro ; VCLMUL_1_TO_8_STEP1 + +;;; =========================================================================== +;;; =========================================================================== +;;; schoolbook multiply (1 to 8 blocks) - 2nd step +%macro VCLMUL_1_TO_8_STEP2 10 +%define %%KP %1 ; [in] key pointer +%define %%HI %2 ; [out] ZMM ghash high 128bits +%define %%LO %3 ; [in/out] ZMM ciphered blocks 0 to 3 (in); ghash low 128bits (out) +%define %%TMP0 %4 ; [clobbered] ZMM temporary +%define %%TMP1 %5 ; [clobbered] ZMM temporary +%define %%TMP2 %6 ; [clobbered] ZMM temporary +%define %%TH %7 ; [in/clobbered] ZMM high sum +%define %%TM %8 ; [in/clobbered] ZMM medium sum +%define %%TL %9 ; [in/clobbered] ZMM low sum +%define %%NBLOCKS %10 ; [in] number of blocks to ghash (0 to 8) + +%if %%NBLOCKS == 8 + VCLMUL_STEP2 %%KP, %%HI, %%LO, %%TMP0, %%TMP1, %%TMP2, %%TH, %%TM, %%TL +%elif %%NBLOCKS == 7 + vmovdqu64 %%TMP2, [%%KP + HashKey_7] + VCLMUL_STEP2 NULL, %%HI, %%LO, %%TMP0, %%TMP1, %%TMP2, %%TH, %%TM, %%TL, %%TMP2, 4 +%elif %%NBLOCKS == 6 + vmovdqu64 %%TMP2, [%%KP + HashKey_6] + VCLMUL_STEP2 NULL, %%HI, %%LO, %%TMP0, %%TMP1, %%TMP2, %%TH, %%TM, %%TL, %%TMP2, 4 +%elif %%NBLOCKS == 5 + vmovdqu64 %%TMP2, [%%KP + HashKey_5] + VCLMUL_STEP2 NULL, %%HI, %%LO, %%TMP0, %%TMP1, %%TMP2, %%TH, %%TM, %%TL, %%TMP2, 4 +%elif %%NBLOCKS == 4 + vmovdqu64 %%TMP2, [%%KP + HashKey_4] + VCLMUL_STEP2 NULL, %%HI, %%LO, %%TMP0, %%TMP1, %%TMP2, %%TH, %%TM, %%TL, %%TMP2, 4 +%elif %%NBLOCKS == 3 + vmovdqu64 %%TMP2, [%%KP + HashKey_3] + vmovdqa64 %%TMP1, [rel mask_out_top_block] + vpandq %%TMP2, %%TMP1 + vpandq %%LO, %%TMP1 + VCLMUL_STEP2 NULL, %%HI, %%LO, %%TMP0, %%TMP1, %%TMP2, %%TH, %%TM, %%TL, %%TMP2, 4 +%elif %%NBLOCKS == 2 + vmovdqu64 YWORD(%%TMP2), [%%KP + HashKey_2] + VCLMUL_STEP2 NULL, YWORD(%%HI), YWORD(%%LO), \ + YWORD(%%TMP0), YWORD(%%TMP1), YWORD(%%TMP2), \ + YWORD(%%TH), YWORD(%%TM), YWORD(%%TL), YWORD(%%TMP2), 2 +%elif %%NBLOCKS == 1 + vmovdqu64 XWORD(%%TMP2), [%%KP + HashKey_1] + VCLMUL_STEP2 NULL, XWORD(%%HI), XWORD(%%LO), \ + XWORD(%%TMP0), XWORD(%%TMP1), XWORD(%%TMP2), \ + XWORD(%%TH), XWORD(%%TM), XWORD(%%TL), XWORD(%%TMP2), 1 +%else + vpxorq %%HI, %%HI + vpxorq %%LO, %%LO +%endif +%endmacro ; VCLMUL_1_TO_8_STEP2 + +;;; =========================================================================== +;;; =========================================================================== +;;; GHASH 1 to 16 blocks of cipher text +;;; - performs reduction at the end +;;; - can take intermediate GHASH sums as input +%macro GHASH_1_TO_16 20 +%define %%KP %1 ; [in] pointer to expanded keys +%define %%GHASH %2 ; [out] ghash output +%define %%T1 %3 ; [clobbered] temporary ZMM +%define %%T2 %4 ; [clobbered] temporary ZMM +%define %%T3 %5 ; [clobbered] temporary ZMM +%define %%T4 %6 ; [clobbered] temporary ZMM +%define %%T5 %7 ; [clobbered] temporary ZMM +%define %%T6 %8 ; [clobbered] temporary ZMM +%define %%T7 %9 ; [clobbered] temporary ZMM +%define %%T8 %10 ; [clobbered] temporary ZMM +%define %%T9 %11 ; [clobbered] temporary ZMM +%define %%GH %12 ; [in/cloberred] ghash sum (high) or "no_zmm" +%define %%GL %13 ; [in/cloberred] ghash sum (low) or "no_zmm" +%define %%GM %14 ; [in/cloberred] ghash sum (medium) or "no_zmm" +%define %%AAD_HASH_IN %15 ; [in] input hash value +%define %%CIPHER_IN0 %16 ; [in] ZMM with cipher text blocks 0-3 +%define %%CIPHER_IN1 %17 ; [in] ZMM with cipher text blocks 4-7 +%define %%CIPHER_IN2 %18 ; [in] ZMM with cipher text blocks 8-11 +%define %%CIPHER_IN3 %19 ; [in] ZMM with cipher text blocks 12-15 +%define %%NUM_BLOCKS %20 ; [in] numerical value, number of blocks + +%define %%T0H %%T1 +%define %%T0L %%T2 +%define %%T0M1 %%T3 +%define %%T0M2 %%T4 + +%define %%T1H %%T5 +%define %%T1L %%T6 +%define %%T1M1 %%T7 +%define %%T1M2 %%T8 + +%define %%HK %%T9 + +%assign hashk HashKey_ %+ %%NUM_BLOCKS +%assign reg_idx 0 +%assign blocks_left %%NUM_BLOCKS + + vpxorq %%CIPHER_IN0, %%CIPHER_IN0, %%AAD_HASH_IN + +%assign first_result 1 + +%ifnidn %%GH, no_zmm +%ifnidn %%GM, no_zmm +%ifnidn %%GL, no_zmm + ;; GHASH sums passed in to be updated and + ;; reduced at the end + vmovdqa64 %%T0H, %%GH + vmovdqa64 %%T0L, %%GL + vmovdqa64 %%T0M1, %%GM + vpxorq %%T0M2, %%T0M2 +%assign first_result 0 +%endif +%endif +%endif + +%rep (blocks_left / 4) +%xdefine %%REG_IN %%CIPHER_IN %+ reg_idx + vmovdqu64 %%HK, [%%KP + hashk] +%if first_result == 1 + vpclmulqdq %%T0H, %%REG_IN, %%HK, 0x11 ; H = a1*b1 + vpclmulqdq %%T0L, %%REG_IN, %%HK, 0x00 ; L = a0*b0 + vpclmulqdq %%T0M1, %%REG_IN, %%HK, 0x01 ; M1 = a1*b0 + vpclmulqdq %%T0M2, %%REG_IN, %%HK, 0x10 ; TM2 = a0*b1 +%assign first_result 0 +%else + vpclmulqdq %%T1H, %%REG_IN, %%HK, 0x11 ; H = a1*b1 + vpclmulqdq %%T1L, %%REG_IN, %%HK, 0x00 ; L = a0*b0 + vpclmulqdq %%T1M1, %%REG_IN, %%HK, 0x01 ; M1 = a1*b0 + vpclmulqdq %%T1M2, %%REG_IN, %%HK, 0x10 ; M2 = a0*b1 + vpxorq %%T0H, %%T0H, %%T1H + vpxorq %%T0L, %%T0L, %%T1L + vpxorq %%T0M1, %%T0M1, %%T1M1 + vpxorq %%T0M2, %%T0M2, %%T1M2 +%endif +%undef %%REG_IN +%assign reg_idx (reg_idx + 1) +%assign hashk (hashk + 64) +%assign blocks_left (blocks_left - 4) +%endrep + +%if blocks_left > 0 +;; There are 1, 2 or 3 blocks left to process. +;; It may also be that they are the only blocks to process. + +%xdefine %%REG_IN %%CIPHER_IN %+ reg_idx + +%if first_result == 1 +;; Case where %%NUM_BLOCKS = 1, 2 or 3 +%xdefine %%OUT_H %%T0H +%xdefine %%OUT_L %%T0L +%xdefine %%OUT_M1 %%T0M1 +%xdefine %%OUT_M2 %%T0M2 +%else +%xdefine %%OUT_H %%T1H +%xdefine %%OUT_L %%T1L +%xdefine %%OUT_M1 %%T1M1 +%xdefine %%OUT_M2 %%T1M2 +%endif + +%if blocks_left == 1 + vmovdqu64 XWORD(%%HK), [%%KP + hashk] + vpclmulqdq XWORD(%%OUT_H), XWORD(%%REG_IN), XWORD(%%HK), 0x11 ; %%TH = a1*b1 + vpclmulqdq XWORD(%%OUT_L), XWORD(%%REG_IN), XWORD(%%HK), 0x00 ; %%TL = a0*b0 + vpclmulqdq XWORD(%%OUT_M1), XWORD(%%REG_IN), XWORD(%%HK), 0x01 ; %%TM1 = a1*b0 + vpclmulqdq XWORD(%%OUT_M2), XWORD(%%REG_IN), XWORD(%%HK), 0x10 ; %%TM2 = a0*b1 +%elif blocks_left == 2 + vmovdqu64 YWORD(%%HK), [%%KP + hashk] + vpclmulqdq YWORD(%%OUT_H), YWORD(%%REG_IN), YWORD(%%HK), 0x11 ; %%TH = a1*b1 + vpclmulqdq YWORD(%%OUT_L), YWORD(%%REG_IN), YWORD(%%HK), 0x00 ; %%TL = a0*b0 + vpclmulqdq YWORD(%%OUT_M1), YWORD(%%REG_IN), YWORD(%%HK), 0x01 ; %%TM1 = a1*b0 + vpclmulqdq YWORD(%%OUT_M2), YWORD(%%REG_IN), YWORD(%%HK), 0x10 ; %%TM2 = a0*b1 +%else ; blocks_left == 3 + vmovdqu64 YWORD(%%HK), [%%KP + hashk] + vinserti64x2 %%HK, [%%KP + hashk + 32], 2 + vpclmulqdq %%OUT_H, %%REG_IN, %%HK, 0x11 ; %%TH = a1*b1 + vpclmulqdq %%OUT_L, %%REG_IN, %%HK, 0x00 ; %%TL = a0*b0 + vpclmulqdq %%OUT_M1, %%REG_IN, %%HK, 0x01 ; %%TM1 = a1*b0 + vpclmulqdq %%OUT_M2, %%REG_IN, %%HK, 0x10 ; %%TM2 = a0*b1 +%endif ; blocks_left + +%undef %%REG_IN +%undef %%OUT_H +%undef %%OUT_L +%undef %%OUT_M1 +%undef %%OUT_M2 + +%if first_result != 1 + vpxorq %%T0H, %%T0H, %%T1H + vpxorq %%T0L, %%T0L, %%T1L + vpxorq %%T0M1, %%T0M1, %%T1M1 + vpxorq %%T0M2, %%T0M2, %%T1M2 +%endif + +%endif ; blocks_left > 0 + + ;; integrate TM into TH and TL + vpxorq %%T0M1, %%T0M1, %%T0M2 + vpsrldq %%T1M1, %%T0M1, 8 + vpslldq %%T1M2, %%T0M1, 8 + vpxorq %%T0H, %%T0H, %%T1M1 + vpxorq %%T0L, %%T0L, %%T1M2 + + ;; add TH and TL 128-bit words horizontally + VHPXORI4x128 %%T0H, %%T1M1 + VHPXORI4x128 %%T0L, %%T1M2 + + ;; reduction + vmovdqa64 XWORD(%%HK), [rel POLY2] + VCLMUL_REDUCE XWORD(%%GHASH), XWORD(%%HK), \ + XWORD(%%T0H), XWORD(%%T0L), XWORD(%%T0M1), XWORD(%%T0M2) +%endmacro + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;; GHASH_MUL MACRO to implement: Data*HashKey mod (128,127,126,121,0) +;;; Input: A and B (128-bits each, bit-reflected) +;;; Output: C = A*B*x mod poly, (i.e. >>1 ) +;;; To compute GH = GH*HashKey mod poly, give HK = HashKey<<1 mod poly as input +;;; GH = GH * HK * x mod poly which is equivalent to GH*HashKey mod poly. +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +%macro GHASH_MUL 7 +%define %%GH %1 ; 16 Bytes +%define %%HK %2 ; 16 Bytes +%define %%T1 %3 +%define %%T2 %4 +%define %%T3 %5 +%define %%T4 %6 +%define %%T5 %7 + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + + vpclmulqdq %%T1, %%GH, %%HK, 0x11 ; %%T1 = a1*b1 + vpclmulqdq %%T2, %%GH, %%HK, 0x00 ; %%T2 = a0*b0 + vpclmulqdq %%T3, %%GH, %%HK, 0x01 ; %%T3 = a1*b0 + vpclmulqdq %%GH, %%GH, %%HK, 0x10 ; %%GH = a0*b1 + vpxorq %%GH, %%GH, %%T3 + + + vpsrldq %%T3, %%GH, 8 ; shift-R %%GH 2 DWs + vpslldq %%GH, %%GH, 8 ; shift-L %%GH 2 DWs + + vpxorq %%T1, %%T1, %%T3 + vpxorq %%GH, %%GH, %%T2 + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;first phase of the reduction + vmovdqu64 %%T3, [rel POLY2] + + vpclmulqdq %%T2, %%T3, %%GH, 0x01 + vpslldq %%T2, %%T2, 8 ; shift-L %%T2 2 DWs + + vpxorq %%GH, %%GH, %%T2 ; first phase of the reduction complete + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;second phase of the reduction + vpclmulqdq %%T2, %%T3, %%GH, 0x00 + vpsrldq %%T2, %%T2, 4 ; shift-R only 1-DW to obtain 2-DWs shift-R + + vpclmulqdq %%GH, %%T3, %%GH, 0x10 + vpslldq %%GH, %%GH, 4 ; Shift-L 1-DW to obtain result with no shifts + + ; second phase of the reduction complete, the result is in %%GH + vpternlogq %%GH, %%T1, %%T2, 0x96 ; GH = GH xor T1 xor T2 + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +%endmacro + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;; In PRECOMPUTE, the commands filling Hashkey_i_k are not required for avx512 +;;; functions, but are kept to allow users to switch cpu architectures between calls +;;; of pre, init, update, and finalize. +%macro PRECOMPUTE 8 +%define %%GDATA %1 +%define %%HK %2 +%define %%T1 %3 +%define %%T2 %4 +%define %%T3 %5 +%define %%T4 %6 +%define %%T5 %7 +%define %%T6 %8 + + vmovdqa %%T5, %%HK + + ;; GHASH keys 2 to 48 or 128 +%ifdef GCM_BIG_DATA +%assign max_hkey_idx 128 +%else +%assign max_hkey_idx 48 +%endif + +%assign i 2 +%rep (max_hkey_idx - 1) + GHASH_MUL %%T5, %%HK, %%T1, %%T3, %%T4, %%T6, %%T2 ; %%T5 = HashKey^i<<1 mod poly + vmovdqu [%%GDATA + HashKey_ %+ i], %%T5 ; [HashKey_i] = %%T5 +%assign i (i + 1) +%endrep + +%endmacro + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;; READ_SMALL_DATA_INPUT +;;; Packs xmm register with data when data input is less or equal to 16 bytes +;;; Returns 0 if data has length 0 +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +%macro READ_SMALL_DATA_INPUT 5 +%define %%OUTPUT %1 ; [out] xmm register +%define %%INPUT %2 ; [in] buffer pointer to read from +%define %%LENGTH %3 ; [in] number of bytes to read +%define %%TMP1 %4 ; [clobbered] +%define %%MASK %5 ; [out] k1 to k7 register to store the partial block mask + + cmp %%LENGTH, 16 + jge %%_read_small_data_ge16 + lea %%TMP1, [rel byte_len_to_mask_table] +%ifidn __OUTPUT_FORMAT__, win64 + add %%TMP1, %%LENGTH + add %%TMP1, %%LENGTH + kmovw %%MASK, [%%TMP1] +%else + kmovw %%MASK, [%%TMP1 + %%LENGTH*2] +%endif + vmovdqu8 %%OUTPUT{%%MASK}{z}, [%%INPUT] + jmp %%_read_small_data_end +%%_read_small_data_ge16: + VX512LDR %%OUTPUT, [%%INPUT] + mov %%TMP1, 0xffff + kmovq %%MASK, %%TMP1 +%%_read_small_data_end: +%endmacro ; READ_SMALL_DATA_INPUT + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; CALC_AAD_HASH: Calculates the hash of the data which will not be encrypted. +; Input: The input data (A_IN), that data's length (A_LEN), and the hash key (HASH_KEY). +; Output: The hash of the data (AAD_HASH). +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +%macro CALC_AAD_HASH 18 +%define %%A_IN %1 ; [in] AAD text pointer +%define %%A_LEN %2 ; [in] AAD length +%define %%AAD_HASH %3 ; [out] xmm ghash value +%define %%GDATA_KEY %4 ; [in] pointer to keys +%define %%ZT0 %5 ; [clobbered] ZMM register +%define %%ZT1 %6 ; [clobbered] ZMM register +%define %%ZT2 %7 ; [clobbered] ZMM register +%define %%ZT3 %8 ; [clobbered] ZMM register +%define %%ZT4 %9 ; [clobbered] ZMM register +%define %%ZT5 %10 ; [clobbered] ZMM register +%define %%ZT6 %11 ; [clobbered] ZMM register +%define %%ZT7 %12 ; [clobbered] ZMM register +%define %%ZT8 %13 ; [clobbered] ZMM register +%define %%ZT9 %14 ; [clobbered] ZMM register +%define %%T1 %15 ; [clobbered] GP register +%define %%T2 %16 ; [clobbered] GP register +%define %%T3 %17 ; [clobbered] GP register +%define %%MASKREG %18 ; [clobbered] mask register + +%define %%SHFMSK %%ZT9 +%define %%POLY %%ZT8 +%define %%TH %%ZT7 +%define %%TM %%ZT6 +%define %%TL %%ZT5 + + mov %%T1, %%A_IN ; T1 = AAD + mov %%T2, %%A_LEN ; T2 = aadLen + vpxorq %%AAD_HASH, %%AAD_HASH + + vmovdqa64 %%SHFMSK, [rel SHUF_MASK] + vmovdqa64 %%POLY, [rel POLY2] + +%%_get_AAD_loop128: + cmp %%T2, 128 + jl %%_exit_AAD_loop128 + + vmovdqu64 %%ZT2, [%%T1 + 64*0] ; LO blocks (0-3) + vmovdqu64 %%ZT1, [%%T1 + 64*1] ; HI blocks (4-7) + vpshufb %%ZT2, %%SHFMSK + vpshufb %%ZT1, %%SHFMSK + + vpxorq %%ZT2, %%ZT2, ZWORD(%%AAD_HASH) + + VCLMUL_STEP1 %%GDATA_KEY, %%ZT1, %%ZT0, %%TH, %%TM, %%TL + VCLMUL_STEP2 %%GDATA_KEY, %%ZT1, %%ZT2, %%ZT0, %%ZT3, %%ZT4, %%TH, %%TM, %%TL + + ;; result in %%ZT1(H):%%ZT2(L) + ;; reduce and put the result in AAD_HASH + VCLMUL_REDUCE %%AAD_HASH, XWORD(%%POLY), XWORD(%%ZT1), XWORD(%%ZT2), \ + XWORD(%%ZT0), XWORD(%%ZT3) + + sub %%T2, 128 + je %%_CALC_AAD_done + + add %%T1, 128 + jmp %%_get_AAD_loop128 + +%%_exit_AAD_loop128: + or %%T2, %%T2 + jz %%_CALC_AAD_done + + ;; prep mask source address + lea %%T3, [rel byte64_len_to_mask_table] + lea %%T3, [%%T3 + %%T2*8] + + ;; calculate number of blocks to ghash (including partial bytes) + add %%T2, 15 + and %%T2, -16 ; 1 to 8 blocks possible here + shr %%T2, 4 + cmp %%T2, 7 + je %%_AAD_blocks_7 + cmp %%T2, 6 + je %%_AAD_blocks_6 + cmp %%T2, 5 + je %%_AAD_blocks_5 + cmp %%T2, 4 + je %%_AAD_blocks_4 + cmp %%T2, 3 + je %%_AAD_blocks_3 + cmp %%T2, 2 + je %%_AAD_blocks_2 + cmp %%T2, 1 + je %%_AAD_blocks_1 + ;; fall through for 8 blocks + + ;; The flow of each of these cases is identical: + ;; - load blocks plain text + ;; - shuffle loaded blocks + ;; - xor in current hash value into block 0 + ;; - perform up multiplications with ghash keys + ;; - jump to reduction code +%%_AAD_blocks_8: + sub %%T3, (64 * 8) + kmovq %%MASKREG, [%%T3] + vmovdqu8 %%ZT2, [%%T1 + 64*0] + vmovdqu8 %%ZT1{%%MASKREG}{z}, [%%T1 + 64*1] + vpshufb %%ZT2, %%SHFMSK + vpshufb %%ZT1, %%SHFMSK + vpxorq %%ZT2, %%ZT2, ZWORD(%%AAD_HASH) ; xor in current ghash + VCLMUL_1_TO_8_STEP1 %%GDATA_KEY, %%ZT1, %%ZT0, %%ZT3, %%TH, %%TM, %%TL, 8 + VCLMUL_1_TO_8_STEP2 %%GDATA_KEY, %%ZT1, %%ZT2, \ + %%ZT0, %%ZT3, %%ZT4, \ + %%TH, %%TM, %%TL, 8 + jmp %%_AAD_blocks_done + +%%_AAD_blocks_7: + sub %%T3, (64 * 8) + kmovq %%MASKREG, [%%T3] + vmovdqu8 %%ZT2, [%%T1 + 64*0] + vmovdqu8 %%ZT1{%%MASKREG}{z}, [%%T1 + 64*1] + vpshufb %%ZT2, %%SHFMSK + vpshufb %%ZT1, %%SHFMSK + vpxorq %%ZT2, %%ZT2, ZWORD(%%AAD_HASH) ; xor in current ghash + VCLMUL_1_TO_8_STEP1 %%GDATA_KEY, %%ZT1, %%ZT0, %%ZT3, %%TH, %%TM, %%TL, 7 + VCLMUL_1_TO_8_STEP2 %%GDATA_KEY, %%ZT1, %%ZT2, \ + %%ZT0, %%ZT3, %%ZT4, \ + %%TH, %%TM, %%TL, 7 + jmp %%_AAD_blocks_done + +%%_AAD_blocks_6: + sub %%T3, (64 * 8) + kmovq %%MASKREG, [%%T3] + vmovdqu8 %%ZT2, [%%T1 + 64*0] + vmovdqu8 YWORD(%%ZT1){%%MASKREG}{z}, [%%T1 + 64*1] + vpshufb %%ZT2, %%SHFMSK + vpshufb YWORD(%%ZT1), YWORD(%%SHFMSK) + vpxorq %%ZT2, %%ZT2, ZWORD(%%AAD_HASH) + VCLMUL_1_TO_8_STEP1 %%GDATA_KEY, %%ZT1, %%ZT0, %%ZT3, %%TH, %%TM, %%TL, 6 + VCLMUL_1_TO_8_STEP2 %%GDATA_KEY, %%ZT1, %%ZT2, \ + %%ZT0, %%ZT3, %%ZT4, \ + %%TH, %%TM, %%TL, 6 + jmp %%_AAD_blocks_done + +%%_AAD_blocks_5: + sub %%T3, (64 * 8) + kmovq %%MASKREG, [%%T3] + vmovdqu8 %%ZT2, [%%T1 + 64*0] + vmovdqu8 XWORD(%%ZT1){%%MASKREG}{z}, [%%T1 + 64*1] + vpshufb %%ZT2, %%SHFMSK + vpshufb XWORD(%%ZT1), XWORD(%%SHFMSK) + vpxorq %%ZT2, %%ZT2, ZWORD(%%AAD_HASH) + VCLMUL_1_TO_8_STEP1 %%GDATA_KEY, %%ZT1, %%ZT0, %%ZT3, %%TH, %%TM, %%TL, 5 + VCLMUL_1_TO_8_STEP2 %%GDATA_KEY, %%ZT1, %%ZT2, \ + %%ZT0, %%ZT3, %%ZT4, \ + %%TH, %%TM, %%TL, 5 + jmp %%_AAD_blocks_done + +%%_AAD_blocks_4: + kmovq %%MASKREG, [%%T3] + vmovdqu8 %%ZT2{%%MASKREG}{z}, [%%T1 + 64*0] + vpshufb %%ZT2, %%SHFMSK + vpxorq %%ZT2, %%ZT2, ZWORD(%%AAD_HASH) + VCLMUL_1_TO_8_STEP1 %%GDATA_KEY, %%ZT1, %%ZT0, %%ZT3, %%TH, %%TM, %%TL, 4 + VCLMUL_1_TO_8_STEP2 %%GDATA_KEY, %%ZT1, %%ZT2, \ + %%ZT0, %%ZT3, %%ZT4, \ + %%TH, %%TM, %%TL, 4 + jmp %%_AAD_blocks_done + +%%_AAD_blocks_3: + kmovq %%MASKREG, [%%T3] + vmovdqu8 %%ZT2{%%MASKREG}{z}, [%%T1 + 64*0] + vpshufb %%ZT2, %%SHFMSK + vpxorq %%ZT2, %%ZT2, ZWORD(%%AAD_HASH) + VCLMUL_1_TO_8_STEP1 %%GDATA_KEY, %%ZT1, %%ZT0, %%ZT3, %%TH, %%TM, %%TL, 3 + VCLMUL_1_TO_8_STEP2 %%GDATA_KEY, %%ZT1, %%ZT2, \ + %%ZT0, %%ZT3, %%ZT4, \ + %%TH, %%TM, %%TL, 3 + jmp %%_AAD_blocks_done + +%%_AAD_blocks_2: + kmovq %%MASKREG, [%%T3] + vmovdqu8 YWORD(%%ZT2){%%MASKREG}{z}, [%%T1 + 64*0] + vpshufb YWORD(%%ZT2), YWORD(%%SHFMSK) + vpxorq %%ZT2, %%ZT2, ZWORD(%%AAD_HASH) + VCLMUL_1_TO_8_STEP1 %%GDATA_KEY, %%ZT1, %%ZT0, %%ZT3, %%TH, %%TM, %%TL, 2 + VCLMUL_1_TO_8_STEP2 %%GDATA_KEY, %%ZT1, %%ZT2, \ + %%ZT0, %%ZT3, %%ZT4, \ + %%TH, %%TM, %%TL, 2 + jmp %%_AAD_blocks_done + +%%_AAD_blocks_1: + kmovq %%MASKREG, [%%T3] + vmovdqu8 XWORD(%%ZT2){%%MASKREG}{z}, [%%T1 + 64*0] + vpshufb XWORD(%%ZT2), XWORD(%%SHFMSK) + vpxorq %%ZT2, %%ZT2, ZWORD(%%AAD_HASH) + VCLMUL_1_TO_8_STEP1 %%GDATA_KEY, %%ZT1, %%ZT0, %%ZT3, %%TH, %%TM, %%TL, 1 + VCLMUL_1_TO_8_STEP2 %%GDATA_KEY, %%ZT1, %%ZT2, \ + %%ZT0, %%ZT3, %%ZT4, \ + %%TH, %%TM, %%TL, 1 + +%%_AAD_blocks_done: + ;; Multiplications have been done. Do the reduction now + VCLMUL_REDUCE %%AAD_HASH, XWORD(%%POLY), XWORD(%%ZT1), XWORD(%%ZT2), \ + XWORD(%%ZT0), XWORD(%%ZT3) +%%_CALC_AAD_done: + ;; result in AAD_HASH + +%endmacro ; CALC_AAD_HASH + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;; PARTIAL_BLOCK +;;; Handles encryption/decryption and the tag partial blocks between +;;; update calls. +;;; Requires the input data be at least 1 byte long. +;;; Output: +;;; A cipher/plain of the first partial block (CYPH_PLAIN_OUT), +;;; AAD_HASH and updated GDATA_CTX +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +%macro PARTIAL_BLOCK 22 +%define %%GDATA_KEY %1 ; [in] key pointer +%define %%GDATA_CTX %2 ; [in] context pointer +%define %%CYPH_PLAIN_OUT %3 ; [in] output buffer +%define %%PLAIN_CYPH_IN %4 ; [in] input buffer +%define %%PLAIN_CYPH_LEN %5 ; [in] buffer length +%define %%DATA_OFFSET %6 ; [in/out] data offset (gets updated) +%define %%AAD_HASH %7 ; [out] updated GHASH value +%define %%ENC_DEC %8 ; [in] cipher direction +%define %%GPTMP0 %9 ; [clobbered] GP temporary register +%define %%GPTMP1 %10 ; [clobbered] GP temporary register +%define %%GPTMP2 %11 ; [clobbered] GP temporary register +%define %%ZTMP0 %12 ; [clobbered] ZMM temporary register +%define %%ZTMP1 %13 ; [clobbered] ZMM temporary register +%define %%ZTMP2 %14 ; [clobbered] ZMM temporary register +%define %%ZTMP3 %15 ; [clobbered] ZMM temporary register +%define %%ZTMP4 %16 ; [clobbered] ZMM temporary register +%define %%ZTMP5 %17 ; [clobbered] ZMM temporary register +%define %%ZTMP6 %18 ; [clobbered] ZMM temporary register +%define %%ZTMP7 %19 ; [clobbered] ZMM temporary register +%define %%ZTMP8 %20 ; [clobbered] ZMM temporary register +%define %%ZTMP9 %21 ; [clobbered] ZMM temporary register +%define %%MASKREG %22 ; [clobbered] mask temporary register + +%define %%XTMP0 XWORD(%%ZTMP0) +%define %%XTMP1 XWORD(%%ZTMP1) +%define %%XTMP2 XWORD(%%ZTMP2) +%define %%XTMP3 XWORD(%%ZTMP3) +%define %%XTMP4 XWORD(%%ZTMP4) +%define %%XTMP5 XWORD(%%ZTMP5) +%define %%XTMP6 XWORD(%%ZTMP6) +%define %%XTMP7 XWORD(%%ZTMP7) +%define %%XTMP8 XWORD(%%ZTMP8) +%define %%XTMP9 XWORD(%%ZTMP9) + +%define %%LENGTH %%GPTMP0 +%define %%IA0 %%GPTMP1 +%define %%IA1 %%GPTMP2 + + mov %%LENGTH, [%%GDATA_CTX + PBlockLen] + or %%LENGTH, %%LENGTH + je %%_partial_block_done ;Leave Macro if no partial blocks + + READ_SMALL_DATA_INPUT %%XTMP0, %%PLAIN_CYPH_IN, %%PLAIN_CYPH_LEN, %%IA0, %%MASKREG + + ;; XTMP1 = my_ctx_data.partial_block_enc_key + vmovdqu64 %%XTMP1, [%%GDATA_CTX + PBlockEncKey] + vmovdqu64 %%XTMP2, [%%GDATA_KEY + HashKey] + + ;; adjust the shuffle mask pointer to be able to shift right %%LENGTH bytes + ;; (16 - %%LENGTH) is the number of bytes in plaintext mod 16) + lea %%IA0, [rel SHIFT_MASK] + add %%IA0, %%LENGTH + vmovdqu64 %%XTMP3, [%%IA0] ; shift right shuffle mask + vpshufb %%XTMP1, %%XTMP3 + +%ifidn %%ENC_DEC, DEC + ;; keep copy of cipher text in %%XTMP4 + vmovdqa64 %%XTMP4, %%XTMP0 +%endif + vpxorq %%XTMP1, %%XTMP0 ; Cyphertext XOR E(K, Yn) + + ;; Set %%IA1 to be the amount of data left in CYPH_PLAIN_IN after filling the block + ;; Determine if partial block is not being filled and shift mask accordingly + mov %%IA1, %%PLAIN_CYPH_LEN + add %%IA1, %%LENGTH + sub %%IA1, 16 + jge %%_no_extra_mask + sub %%IA0, %%IA1 +%%_no_extra_mask: + ;; get the appropriate mask to mask out bottom %%LENGTH bytes of %%XTMP1 + ;; - mask out bottom %%LENGTH bytes of %%XTMP1 + vmovdqu64 %%XTMP0, [%%IA0 + ALL_F - SHIFT_MASK] + vpand %%XTMP1, %%XTMP0 + +%ifidn %%ENC_DEC, DEC + vpand %%XTMP4, %%XTMP0 + vpshufb %%XTMP4, [rel SHUF_MASK] + vpshufb %%XTMP4, %%XTMP3 + vpxorq %%AAD_HASH, %%XTMP4 +%else + vpshufb %%XTMP1, [rel SHUF_MASK] + vpshufb %%XTMP1, %%XTMP3 + vpxorq %%AAD_HASH, %%XTMP1 +%endif + cmp %%IA1, 0 + jl %%_partial_incomplete + + ;; GHASH computation for the last <16 Byte block + GHASH_MUL %%AAD_HASH, %%XTMP2, %%XTMP5, %%XTMP6, %%XTMP7, %%XTMP8, %%XTMP9 + + mov qword [%%GDATA_CTX + PBlockLen], 0 + + ;; Set %%IA1 to be the number of bytes to write out + mov %%IA0, %%LENGTH + mov %%LENGTH, 16 + sub %%LENGTH, %%IA0 + jmp %%_enc_dec_done + +%%_partial_incomplete: +%ifidn __OUTPUT_FORMAT__, win64 + mov %%IA0, %%PLAIN_CYPH_LEN + add [%%GDATA_CTX + PBlockLen], %%IA0 +%else + add [%%GDATA_CTX + PBlockLen], %%PLAIN_CYPH_LEN +%endif + mov %%LENGTH, %%PLAIN_CYPH_LEN + +%%_enc_dec_done: + ;; output encrypted Bytes + + lea %%IA0, [rel byte_len_to_mask_table] + kmovw %%MASKREG, [%%IA0 + %%LENGTH*2] + vmovdqu64 [%%GDATA_CTX + AadHash], %%AAD_HASH + +%ifidn %%ENC_DEC, ENC + ;; shuffle XTMP1 back to output as ciphertext + vpshufb %%XTMP1, [rel SHUF_MASK] + vpshufb %%XTMP1, %%XTMP3 +%endif + vmovdqu8 [%%CYPH_PLAIN_OUT + %%DATA_OFFSET]{%%MASKREG}, %%XTMP1 + add %%DATA_OFFSET, %%LENGTH +%%_partial_block_done: +%endmacro ; PARTIAL_BLOCK + + +%macro GHASH_SINGLE_MUL 9 +%define %%GDATA %1 +%define %%HASHKEY %2 +%define %%CIPHER %3 +%define %%STATE_11 %4 +%define %%STATE_00 %5 +%define %%STATE_MID %6 +%define %%T1 %7 +%define %%T2 %8 +%define %%FIRST %9 + + vmovdqu %%T1, [%%GDATA + %%HASHKEY] +%ifidn %%FIRST, first + vpclmulqdq %%STATE_11, %%CIPHER, %%T1, 0x11 ; %%T4 = a1*b1 + vpclmulqdq %%STATE_00, %%CIPHER, %%T1, 0x00 ; %%T4_2 = a0*b0 + vpclmulqdq %%STATE_MID, %%CIPHER, %%T1, 0x01 ; %%T6 = a1*b0 + vpclmulqdq %%T2, %%CIPHER, %%T1, 0x10 ; %%T5 = a0*b1 + vpxor %%STATE_MID, %%STATE_MID, %%T2 +%else + vpclmulqdq %%T2, %%CIPHER, %%T1, 0x11 + vpxor %%STATE_11, %%STATE_11, %%T2 + + vpclmulqdq %%T2, %%CIPHER, %%T1, 0x00 + vpxor %%STATE_00, %%STATE_00, %%T2 + + vpclmulqdq %%T2, %%CIPHER, %%T1, 0x01 + vpxor %%STATE_MID, %%STATE_MID, %%T2 + + vpclmulqdq %%T2, %%CIPHER, %%T1, 0x10 + vpxor %%STATE_MID, %%STATE_MID, %%T2 +%endif + +%endmacro + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;; This macro is used to "warm-up" pipeline for GHASH_8_ENCRYPT_8_PARALLEL +;;; macro code. It is called only for data lenghts 128 and above. +;;; The flow is as follows: +;;; - encrypt the initial %%num_initial_blocks blocks (can be 0) +;;; - encrypt the next 8 blocks and stitch with +;;; GHASH for the first %%num_initial_blocks +;;; - the last 8th block can be partial (lengths between 129 and 239) +;;; - partial block ciphering is handled within this macro +;;; - top bytes of such block are cleared for +;;; the subsequent GHASH calculations +;;; - PBlockEncKey needs to be setup in case of multi-call +;;; - top bytes of the block need to include encrypted counter block so that +;;; when handling partial block case text is read and XOR'ed against it. +;;; This needs to be in un-shuffled format. + +%macro INITIAL_BLOCKS 26-27 +%define %%GDATA_KEY %1 ; [in] pointer to GCM keys +%define %%GDATA_CTX %2 ; [in] pointer to GCM context +%define %%CYPH_PLAIN_OUT %3 ; [in] output buffer +%define %%PLAIN_CYPH_IN %4 ; [in] input buffer +%define %%LENGTH %5 ; [in/out] number of bytes to process +%define %%DATA_OFFSET %6 ; [in/out] data offset +%define %%num_initial_blocks %7 ; [in] can be 0, 1, 2, 3, 4, 5, 6 or 7 +%define %%CTR %8 ; [in/out] XMM counter block +%define %%AAD_HASH %9 ; [in/out] ZMM with AAD hash +%define %%ZT1 %10 ; [out] ZMM cipher blocks 0-3 for GHASH +%define %%ZT2 %11 ; [out] ZMM cipher blocks 4-7 for GHASH +%define %%ZT3 %12 ; [clobbered] ZMM temporary +%define %%ZT4 %13 ; [clobbered] ZMM temporary +%define %%ZT5 %14 ; [clobbered] ZMM temporary +%define %%ZT6 %15 ; [clobbered] ZMM temporary +%define %%ZT7 %16 ; [clobbered] ZMM temporary +%define %%ZT8 %17 ; [clobbered] ZMM temporary +%define %%ZT9 %18 ; [clobbered] ZMM temporary +%define %%ZT10 %19 ; [clobbered] ZMM temporary +%define %%ZT11 %20 ; [clobbered] ZMM temporary +%define %%ZT12 %21 ; [clobbered] ZMM temporary +%define %%IA0 %22 ; [clobbered] GP temporary +%define %%IA1 %23 ; [clobbered] GP temporary +%define %%ENC_DEC %24 ; [in] ENC/DEC selector +%define %%MASKREG %25 ; [clobbered] mask register +%define %%SHUFMASK %26 ; [in] ZMM with BE/LE shuffle mask +%define %%PARTIAL_PRESENT %27 ; [in] "no_partial_block" option can be passed here (if length is guaranteed to be > 15*16 bytes) + +%define %%T1 XWORD(%%ZT1) +%define %%T2 XWORD(%%ZT2) +%define %%T3 XWORD(%%ZT3) +%define %%T4 XWORD(%%ZT4) +%define %%T5 XWORD(%%ZT5) +%define %%T6 XWORD(%%ZT6) +%define %%T7 XWORD(%%ZT7) +%define %%T8 XWORD(%%ZT8) +%define %%T9 XWORD(%%ZT9) + +%define %%TH %%ZT10 +%define %%TM %%ZT11 +%define %%TL %%ZT12 + +;; determine if partial block code needs to be added +%assign partial_block_possible 1 +%if %0 > 26 +%ifidn %%PARTIAL_PRESENT, no_partial_block +%assign partial_block_possible 0 +%endif +%endif + +%if %%num_initial_blocks > 0 + ;; prepare AES counter blocks +%if %%num_initial_blocks == 1 + vpaddd %%T3, %%CTR, [rel ONE] +%elif %%num_initial_blocks == 2 + vshufi64x2 YWORD(%%ZT3), YWORD(%%CTR), YWORD(%%CTR), 0 + vpaddd YWORD(%%ZT3), YWORD(%%ZT3), [rel ddq_add_1234] +%else + vshufi64x2 ZWORD(%%CTR), ZWORD(%%CTR), ZWORD(%%CTR), 0 + vpaddd %%ZT3, ZWORD(%%CTR), [rel ddq_add_1234] + vpaddd %%ZT4, ZWORD(%%CTR), [rel ddq_add_5678] +%endif + + ;; extract new counter value (%%T3) + ;; shuffle the counters for AES rounds +%if %%num_initial_blocks <= 4 + vextracti32x4 %%CTR, %%ZT3, (%%num_initial_blocks - 1) +%else + vextracti32x4 %%CTR, %%ZT4, (%%num_initial_blocks - 5) +%endif + ZMM_OPCODE3_DSTR_SRC1R_SRC2R_BLOCKS_0_16 %%num_initial_blocks, vpshufb, \ + %%ZT3, %%ZT4, no_zmm, no_zmm, \ + %%ZT3, %%ZT4, no_zmm, no_zmm, \ + %%SHUFMASK, %%SHUFMASK, %%SHUFMASK, %%SHUFMASK + + ;; load plain/cipher text + ZMM_LOAD_BLOCKS_0_16 %%num_initial_blocks, %%PLAIN_CYPH_IN, %%DATA_OFFSET, \ + %%ZT5, %%ZT6, no_zmm, no_zmm + + ;; AES rounds and XOR with plain/cipher text +%assign j 0 +%rep (NROUNDS + 2) + vbroadcastf64x2 %%ZT1, [%%GDATA_KEY + (j * 16)] + ZMM_AESENC_ROUND_BLOCKS_0_16 %%ZT3, %%ZT4, no_zmm, no_zmm, \ + %%ZT1, j, \ + %%ZT5, %%ZT6, no_zmm, no_zmm, \ + %%num_initial_blocks, NROUNDS +%assign j (j + 1) +%endrep + + ;; write cipher/plain text back to output and + ;; zero bytes outside the mask before hashing + ZMM_STORE_BLOCKS_0_16 %%num_initial_blocks, %%CYPH_PLAIN_OUT, %%DATA_OFFSET, \ + %%ZT3, %%ZT4, no_zmm, no_zmm + + ;; Shuffle the cipher text blocks for hashing part + ;; ZT5 and ZT6 are expected outputs with blocks for hashing +%ifidn %%ENC_DEC, DEC + ;; Decrypt case + ;; - cipher blocks are in ZT5 & ZT6 + ZMM_OPCODE3_DSTR_SRC1R_SRC2R_BLOCKS_0_16 %%num_initial_blocks, vpshufb, \ + %%ZT5, %%ZT6, no_zmm, no_zmm, \ + %%ZT5, %%ZT6, no_zmm, no_zmm, \ + %%SHUFMASK, %%SHUFMASK, %%SHUFMASK, %%SHUFMASK +%else + ;; Encrypt case + ;; - cipher blocks are in ZT3 & ZT4 + ZMM_OPCODE3_DSTR_SRC1R_SRC2R_BLOCKS_0_16 %%num_initial_blocks, vpshufb, \ + %%ZT5, %%ZT6, no_zmm, no_zmm, \ + %%ZT3, %%ZT4, no_zmm, no_zmm, \ + %%SHUFMASK, %%SHUFMASK, %%SHUFMASK, %%SHUFMASK +%endif ; Encrypt + + ;; adjust data offset and length + sub %%LENGTH, (%%num_initial_blocks * 16) + add %%DATA_OFFSET, (%%num_initial_blocks * 16) + + ;; At this stage + ;; - ZT5:ZT6 include cipher blocks to be GHASH'ed + +%endif ; %%num_initial_blocks > 0 + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; - cipher of %%num_initial_blocks is done + ;; - prepare counter blocks for the next 8 blocks (ZT3 & ZT4) + ;; - save the last block in %%CTR + ;; - shuffle the blocks for AES + ;; - stitch encryption of the new blocks with + ;; GHASHING the previous blocks + vshufi64x2 ZWORD(%%CTR), ZWORD(%%CTR), ZWORD(%%CTR), 0 + vpaddd %%ZT3, ZWORD(%%CTR), [rel ddq_add_1234] + vpaddd %%ZT4, ZWORD(%%CTR), [rel ddq_add_5678] + vextracti32x4 %%CTR, %%ZT4, 3 + + vpshufb %%ZT3, %%SHUFMASK + vpshufb %%ZT4, %%SHUFMASK + +%if partial_block_possible != 0 + ;; get text load/store mask (assume full mask by default) + mov %%IA0, 0xffff_ffff_ffff_ffff +%if %%num_initial_blocks > 0 + ;; NOTE: 'jge' is always taken for %%num_initial_blocks = 0 + ;; This macro is executed for lenght 128 and up, + ;; zero length is checked in GCM_ENC_DEC. + ;; We know there is partial block if: + ;; LENGTH - 16*num_initial_blocks < 128 + cmp %%LENGTH, 128 + jge %%_initial_partial_block_continue + mov %%IA1, rcx + mov rcx, 128 + sub rcx, %%LENGTH + shr %%IA0, cl + mov rcx, %%IA1 +%%_initial_partial_block_continue: +%endif + kmovq %%MASKREG, %%IA0 + ;; load plain or cipher text (masked) + ZMM_LOAD_MASKED_BLOCKS_0_16 8, %%PLAIN_CYPH_IN, %%DATA_OFFSET, \ + %%ZT1, %%ZT2, no_zmm, no_zmm, %%MASKREG +%else + ;; load plain or cipher text + ZMM_LOAD_BLOCKS_0_16 8, %%PLAIN_CYPH_IN, %%DATA_OFFSET, \ + %%ZT1, %%ZT2, no_zmm, no_zmm +%endif ;; partial_block_possible + + ;; === AES ROUND 0 +%assign aes_round 0 + vbroadcastf64x2 %%ZT8, [%%GDATA_KEY + (aes_round * 16)] + ZMM_AESENC_ROUND_BLOCKS_0_16 %%ZT3, %%ZT4, no_zmm, no_zmm, \ + %%ZT8, aes_round, \ + %%ZT1, %%ZT2, no_zmm, no_zmm, \ + 8, NROUNDS +%assign aes_round (aes_round + 1) + + ;; === GHASH blocks 4-7 +%if (%%num_initial_blocks > 0) + ;; Hash in AES state + vpxorq %%ZT5, %%ZT5, %%AAD_HASH + + VCLMUL_1_TO_8_STEP1 %%GDATA_KEY, %%ZT6, %%ZT8, %%ZT9, \ + %%TH, %%TM, %%TL, %%num_initial_blocks +%endif + + ;; === [1/3] of AES rounds + +%rep ((NROUNDS + 1) / 3) + vbroadcastf64x2 %%ZT8, [%%GDATA_KEY + (aes_round * 16)] + ZMM_AESENC_ROUND_BLOCKS_0_16 %%ZT3, %%ZT4, no_zmm, no_zmm, \ + %%ZT8, aes_round, \ + %%ZT1, %%ZT2, no_zmm, no_zmm, \ + 8, NROUNDS +%assign aes_round (aes_round + 1) +%endrep ; %rep ((NROUNDS + 1) / 2) + + ;; === GHASH blocks 0-3 and gather +%if (%%num_initial_blocks > 0) + VCLMUL_1_TO_8_STEP2 %%GDATA_KEY, %%ZT6, %%ZT5, \ + %%ZT7, %%ZT8, %%ZT9, \ + %%TH, %%TM, %%TL, %%num_initial_blocks +%endif + + ;; === [2/3] of AES rounds + +%rep ((NROUNDS + 1) / 3) + vbroadcastf64x2 %%ZT8, [%%GDATA_KEY + (aes_round * 16)] + ZMM_AESENC_ROUND_BLOCKS_0_16 %%ZT3, %%ZT4, no_zmm, no_zmm, \ + %%ZT8, aes_round, \ + %%ZT1, %%ZT2, no_zmm, no_zmm, \ + 8, NROUNDS +%assign aes_round (aes_round + 1) +%endrep ; %rep ((NROUNDS + 1) / 2) + + ;; === GHASH reduction + +%if (%%num_initial_blocks > 0) + ;; [out] AAD_HASH - hash output + ;; [in] T8 - polynomial + ;; [in] T6 - high, T5 - low + ;; [clobbered] T9, T7 - temporary + vmovdqu64 %%T8, [rel POLY2] + VCLMUL_REDUCE XWORD(%%AAD_HASH), %%T8, %%T6, %%T5, %%T7, %%T9 +%endif + + ;; === [3/3] of AES rounds + +%rep (((NROUNDS + 1) / 3) + 2) +%if aes_round < (NROUNDS + 2) + vbroadcastf64x2 %%ZT8, [%%GDATA_KEY + (aes_round * 16)] + ZMM_AESENC_ROUND_BLOCKS_0_16 %%ZT3, %%ZT4, no_zmm, no_zmm, \ + %%ZT8, aes_round, \ + %%ZT1, %%ZT2, no_zmm, no_zmm, \ + 8, NROUNDS +%assign aes_round (aes_round + 1) +%endif +%endrep ; %rep ((NROUNDS + 1) / 2) + +%if partial_block_possible != 0 + ;; write cipher/plain text back to output and + ;; zero bytes outside the mask before hashing + ZMM_STORE_MASKED_BLOCKS_0_16 8, %%CYPH_PLAIN_OUT, %%DATA_OFFSET, \ + %%ZT3, %%ZT4, no_zmm, no_zmm, %%MASKREG + ;; check if there is partial block + cmp %%LENGTH, 128 + jl %%_initial_save_partial + ;; adjust offset and length + add %%DATA_OFFSET, 128 + sub %%LENGTH, 128 + jmp %%_initial_blocks_done +%%_initial_save_partial: + ;; partial block case + ;; - save the partial block in unshuffled format + ;; - ZT4 is partially XOR'ed with data and top bytes contain + ;; encrypted counter block only + ;; - save number of bytes process in the partial block + ;; - adjust offset and zero the length + ;; - clear top bytes of the partial block for subsequent GHASH calculations + vextracti32x4 [%%GDATA_CTX + PBlockEncKey], %%ZT4, 3 + add %%DATA_OFFSET, %%LENGTH + sub %%LENGTH, (128 - 16) + mov [%%GDATA_CTX + PBlockLen], %%LENGTH + xor %%LENGTH, %%LENGTH + vmovdqu8 %%ZT4{%%MASKREG}{z}, %%ZT4 +%%_initial_blocks_done: +%else + ZMM_STORE_BLOCKS_0_16 8, %%CYPH_PLAIN_OUT, %%DATA_OFFSET, \ + %%ZT3, %%ZT4, no_zmm, no_zmm + add %%DATA_OFFSET, 128 + sub %%LENGTH, 128 +%endif ;; partial_block_possible + + ;; Shuffle AES result for GHASH. +%ifidn %%ENC_DEC, DEC + ;; Decrypt case + ;; - cipher blocks are in ZT1 & ZT2 + vpshufb %%ZT1, %%SHUFMASK + vpshufb %%ZT2, %%SHUFMASK +%else + ;; Encrypt case + ;; - cipher blocks are in ZT3 & ZT4 + vpshufb %%ZT1, %%ZT3, %%SHUFMASK + vpshufb %%ZT2, %%ZT4, %%SHUFMASK +%endif ; Encrypt + + ;; Current hash value is in AAD_HASH + + ;; Combine GHASHed value with the corresponding ciphertext + vpxorq %%ZT1, %%ZT1, %%AAD_HASH + +%endmacro ; INITIAL_BLOCKS +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;; INITIAL_BLOCKS_PARTIAL macro with support for a partial final block. +;;; It may look similar to INITIAL_BLOCKS but its usage is different: +;;; - first encrypts/decrypts required number of blocks and then +;;; ghashes these blocks +;;; - Small packets or left over data chunks (<256 bytes) +;;; - single or multi call +;;; - Remaining data chunks below 256 bytes (multi buffer code) +;;; +;;; num_initial_blocks is expected to include the partial final block +;;; in the count. +%macro INITIAL_BLOCKS_PARTIAL 41 +%define %%GDATA_KEY %1 ; [in] key pointer +%define %%GDATA_CTX %2 ; [in] context pointer +%define %%CYPH_PLAIN_OUT %3 ; [in] text out pointer +%define %%PLAIN_CYPH_IN %4 ; [in] text out pointer +%define %%LENGTH %5 ; [in/clobbered] length in bytes +%define %%DATA_OFFSET %6 ; [in/out] current data offset (updated) +%define %%num_initial_blocks %7 ; [in] can only be 1, 2, 3, 4, 5, ..., 15 or 16 (not 0) +%define %%CTR %8 ; [in/out] current counter value +%define %%HASH_IN_OUT %9 ; [in/out] XMM ghash in/out value +%define %%ENC_DEC %10 ; [in] cipher direction (ENC/DEC) +%define %%INSTANCE_TYPE %11 ; [in] multi_call or single_call +%define %%ZT0 %12 ; [clobbered] ZMM temporary +%define %%ZT1 %13 ; [clobbered] ZMM temporary +%define %%ZT2 %14 ; [clobbered] ZMM temporary +%define %%ZT3 %15 ; [clobbered] ZMM temporary +%define %%ZT4 %16 ; [clobbered] ZMM temporary +%define %%ZT5 %17 ; [clobbered] ZMM temporary +%define %%ZT6 %18 ; [clobbered] ZMM temporary +%define %%ZT7 %19 ; [clobbered] ZMM temporary +%define %%ZT8 %20 ; [clobbered] ZMM temporary +%define %%ZT9 %21 ; [clobbered] ZMM temporary +%define %%ZT10 %22 ; [clobbered] ZMM temporary +%define %%ZT11 %23 ; [clobbered] ZMM temporary +%define %%ZT12 %24 ; [clobbered] ZMM temporary +%define %%ZT13 %25 ; [clobbered] ZMM temporary +%define %%ZT14 %26 ; [clobbered] ZMM temporary +%define %%ZT15 %27 ; [clobbered] ZMM temporary +%define %%ZT16 %28 ; [clobbered] ZMM temporary +%define %%ZT17 %29 ; [clobbered] ZMM temporary +%define %%ZT18 %30 ; [clobbered] ZMM temporary +%define %%ZT19 %31 ; [clobbered] ZMM temporary +%define %%ZT20 %32 ; [clobbered] ZMM temporary +%define %%ZT21 %33 ; [clobbered] ZMM temporary +%define %%ZT22 %34 ; [clobbered] ZMM temporary +%define %%GH %35 ; [in] ZMM ghash sum (high) +%define %%GL %36 ; [in] ZMM ghash sum (low) +%define %%GM %37 ; [in] ZMM ghash sum (middle) +%define %%IA0 %38 ; [clobbered] GP temporary +%define %%IA1 %39 ; [clobbered] GP temporary +%define %%MASKREG %40 ; [clobbered] mask register +%define %%SHUFMASK %41 ; [in] ZMM with BE/LE shuffle mask + +%define %%T1 XWORD(%%ZT1) +%define %%T2 XWORD(%%ZT2) +%define %%T7 XWORD(%%ZT7) + +%define %%CTR0 %%ZT3 +%define %%CTR1 %%ZT4 +%define %%CTR2 %%ZT8 +%define %%CTR3 %%ZT9 + +%define %%DAT0 %%ZT5 +%define %%DAT1 %%ZT6 +%define %%DAT2 %%ZT10 +%define %%DAT3 %%ZT11 + +%ifnidn %%GH, no_zmm +%ifnidn %%GL, no_zmm +%ifnidn %%GM, no_zmm + ;; when temporary sums are passed then zero HASH IN value + ;; - whatever it holds it is invalid in this case + vpxorq %%HASH_IN_OUT, %%HASH_IN_OUT +%endif +%endif +%endif + ;; Copy ghash to temp reg + vmovdqa64 %%T2, %%HASH_IN_OUT + + ;; prepare AES counter blocks +%if %%num_initial_blocks == 1 + vpaddd XWORD(%%CTR0), %%CTR, [rel ONE] +%elif %%num_initial_blocks == 2 + vshufi64x2 YWORD(%%CTR0), YWORD(%%CTR), YWORD(%%CTR), 0 + vpaddd YWORD(%%CTR0), YWORD(%%CTR0), [rel ddq_add_1234] +%else + vshufi64x2 ZWORD(%%CTR), ZWORD(%%CTR), ZWORD(%%CTR), 0 + vpaddd %%CTR0, ZWORD(%%CTR), [rel ddq_add_1234] +%if %%num_initial_blocks > 4 + vpaddd %%CTR1, ZWORD(%%CTR), [rel ddq_add_5678] +%endif +%if %%num_initial_blocks > 8 + vpaddd %%CTR2, %%CTR0, [rel ddq_add_8888] +%endif +%if %%num_initial_blocks > 12 + vpaddd %%CTR3, %%CTR1, [rel ddq_add_8888] +%endif +%endif + + ;; get load/store mask + lea %%IA0, [rel byte64_len_to_mask_table] + mov %%IA1, %%LENGTH +%if %%num_initial_blocks > 12 + sub %%IA1, 3 * 64 +%elif %%num_initial_blocks > 8 + sub %%IA1, 2 * 64 +%elif %%num_initial_blocks > 4 + sub %%IA1, 64 +%endif + kmovq %%MASKREG, [%%IA0 + %%IA1*8] + + ;; extract new counter value + ;; shuffle the counters for AES rounds +%if %%num_initial_blocks <= 4 + vextracti32x4 %%CTR, %%CTR0, (%%num_initial_blocks - 1) +%elif %%num_initial_blocks <= 8 + vextracti32x4 %%CTR, %%CTR1, (%%num_initial_blocks - 5) +%elif %%num_initial_blocks <= 12 + vextracti32x4 %%CTR, %%CTR2, (%%num_initial_blocks - 9) +%else + vextracti32x4 %%CTR, %%CTR3, (%%num_initial_blocks - 13) +%endif + ZMM_OPCODE3_DSTR_SRC1R_SRC2R_BLOCKS_0_16 %%num_initial_blocks, vpshufb, \ + %%CTR0, %%CTR1, %%CTR2, %%CTR3, \ + %%CTR0, %%CTR1, %%CTR2, %%CTR3, \ + %%SHUFMASK, %%SHUFMASK, %%SHUFMASK, %%SHUFMASK + + ;; load plain/cipher text + ZMM_LOAD_MASKED_BLOCKS_0_16 %%num_initial_blocks, %%PLAIN_CYPH_IN, %%DATA_OFFSET, \ + %%DAT0, %%DAT1, %%DAT2, %%DAT3, %%MASKREG + + ;; AES rounds and XOR with plain/cipher text +%assign j 0 +%rep (NROUNDS + 2) + vbroadcastf64x2 %%ZT1, [%%GDATA_KEY + (j * 16)] + ZMM_AESENC_ROUND_BLOCKS_0_16 %%CTR0, %%CTR1, %%CTR2, %%CTR3, \ + %%ZT1, j, \ + %%DAT0, %%DAT1, %%DAT2, %%DAT3, \ + %%num_initial_blocks, NROUNDS +%assign j (j + 1) +%endrep + + ;; retrieve the last cipher counter block (partially XOR'ed with text) + ;; - this is needed for partial block cases +%if %%num_initial_blocks <= 4 + vextracti32x4 %%T1, %%CTR0, (%%num_initial_blocks - 1) +%elif %%num_initial_blocks <= 8 + vextracti32x4 %%T1, %%CTR1, (%%num_initial_blocks - 5) +%elif %%num_initial_blocks <= 12 + vextracti32x4 %%T1, %%CTR2, (%%num_initial_blocks - 9) +%else + vextracti32x4 %%T1, %%CTR3, (%%num_initial_blocks - 13) +%endif + + ;; write cipher/plain text back to output and + ZMM_STORE_MASKED_BLOCKS_0_16 %%num_initial_blocks, %%CYPH_PLAIN_OUT, %%DATA_OFFSET, \ + %%CTR0, %%CTR1, %%CTR2, %%CTR3, %%MASKREG + + ;; zero bytes outside the mask before hashing +%if %%num_initial_blocks <= 4 + vmovdqu8 %%CTR0{%%MASKREG}{z}, %%CTR0 +%elif %%num_initial_blocks <= 8 + vmovdqu8 %%CTR1{%%MASKREG}{z}, %%CTR1 +%elif %%num_initial_blocks <= 12 + vmovdqu8 %%CTR2{%%MASKREG}{z}, %%CTR2 +%else + vmovdqu8 %%CTR3{%%MASKREG}{z}, %%CTR3 +%endif + + ;; Shuffle the cipher text blocks for hashing part + ;; ZT5 and ZT6 are expected outputs with blocks for hashing +%ifidn %%ENC_DEC, DEC + ;; Decrypt case + ;; - cipher blocks are in ZT5 & ZT6 + ZMM_OPCODE3_DSTR_SRC1R_SRC2R_BLOCKS_0_16 %%num_initial_blocks, vpshufb, \ + %%DAT0, %%DAT1, %%DAT2, %%DAT3, \ + %%DAT0, %%DAT1, %%DAT2, %%DAT3, \ + %%SHUFMASK, %%SHUFMASK, %%SHUFMASK, %%SHUFMASK +%else + ;; Encrypt case + ;; - cipher blocks are in CTR0-CTR3 + ZMM_OPCODE3_DSTR_SRC1R_SRC2R_BLOCKS_0_16 %%num_initial_blocks, vpshufb, \ + %%DAT0, %%DAT1, %%DAT2, %%DAT3, \ + %%CTR0, %%CTR1, %%CTR2, %%CTR3, \ + %%SHUFMASK, %%SHUFMASK, %%SHUFMASK, %%SHUFMASK +%endif ; Encrypt + + ;; Extract the last block for partials and multi_call cases +%if %%num_initial_blocks <= 4 + vextracti32x4 %%T7, %%DAT0, %%num_initial_blocks - 1 +%elif %%num_initial_blocks <= 8 + vextracti32x4 %%T7, %%DAT1, %%num_initial_blocks - 5 +%elif %%num_initial_blocks <= 12 + vextracti32x4 %%T7, %%DAT2, %%num_initial_blocks - 9 +%else + vextracti32x4 %%T7, %%DAT3, %%num_initial_blocks - 13 +%endif + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;; Hash all but the last block of data +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + + ;; update data offset +%if %%num_initial_blocks > 1 + ;; The final block of data may be <16B + add %%DATA_OFFSET, 16 * (%%num_initial_blocks - 1) + sub %%LENGTH, 16 * (%%num_initial_blocks - 1) +%endif + +%if %%num_initial_blocks < 16 + ;; NOTE: the 'jl' is always taken for num_initial_blocks = 16. + ;; This is run in the context of GCM_ENC_DEC_SMALL for length < 256. + cmp %%LENGTH, 16 + jl %%_small_initial_partial_block + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;; Handle a full length final block - encrypt and hash all blocks +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + + sub %%LENGTH, 16 + add %%DATA_OFFSET, 16 + mov [%%GDATA_CTX + PBlockLen], %%LENGTH + + ;; Hash all of the data + + ;; ZT2 - incoming AAD hash (low 128bits) + ;; ZT12-ZT20 - temporary registers + GHASH_1_TO_16 %%GDATA_KEY, %%HASH_IN_OUT, \ + %%ZT12, %%ZT13, %%ZT14, %%ZT15, %%ZT16, \ + %%ZT17, %%ZT18, %%ZT19, %%ZT20, \ + %%GH, %%GL, %%GM, \ + %%ZT2, %%DAT0, %%DAT1, %%DAT2, %%DAT3, \ + %%num_initial_blocks + + jmp %%_small_initial_compute_done +%endif ; %if %%num_initial_blocks < 16 + +%%_small_initial_partial_block: + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;;; Handle ghash for a <16B final block + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + + ;; In this case if it's a single call to encrypt we can + ;; hash all of the data but if it's an init / update / finalize + ;; series of call we need to leave the last block if it's + ;; less than a full block of data. + + mov [%%GDATA_CTX + PBlockLen], %%LENGTH + ;; %%T1 is ciphered counter block + vmovdqu64 [%%GDATA_CTX + PBlockEncKey], %%T1 + +%ifidn %%INSTANCE_TYPE, multi_call +%assign k (%%num_initial_blocks - 1) +%assign last_block_to_hash 1 +%else +%assign k (%%num_initial_blocks) +%assign last_block_to_hash 0 +%endif + +%if (%%num_initial_blocks > last_block_to_hash) + + ;; ZT12-ZT20 - temporary registers + GHASH_1_TO_16 %%GDATA_KEY, %%HASH_IN_OUT, \ + %%ZT12, %%ZT13, %%ZT14, %%ZT15, %%ZT16, \ + %%ZT17, %%ZT18, %%ZT19, %%ZT20, \ + %%GH, %%GL, %%GM, \ + %%ZT2, %%DAT0, %%DAT1, %%DAT2, %%DAT3, k + + ;; just fall through no jmp needed +%else + ;; Record that a reduction is not needed - + ;; In this case no hashes are computed because there + ;; is only one initial block and it is < 16B in length. + ;; We only need to check if a reduction is needed if + ;; initial_blocks == 1 and init/update/final is being used. + ;; In this case we may just have a partial block, and that + ;; gets hashed in finalize. + +%assign need_for_reduction 1 +%ifidn %%GH, no_zmm +%ifidn %%GL, no_zmm +%ifidn %%GM, no_zmm +;; if %%GH, %%GL & %%GM not passed then reduction is not required +%assign need_for_reduction 0 +%endif +%endif +%endif + +%if need_for_reduction == 0 + ;; The hash should end up in HASH_IN_OUT. + ;; The only way we should get here is if there is + ;; a partial block of data, so xor that into the hash. + vpxorq %%HASH_IN_OUT, %%T2, %%T7 +%else + ;; right - here we have nothing to ghash in the small data but + ;; we have GHASH sums passed through that we need to gather and reduce + + ;; integrate TM into TH and TL + vpsrldq %%ZT12, %%GM, 8 + vpslldq %%ZT13, %%GM, 8 + vpxorq %%GH, %%GH, %%ZT12 + vpxorq %%GL, %%GL, %%ZT13 + + ;; add TH and TL 128-bit words horizontally + VHPXORI4x128 %%GH, %%ZT12 + VHPXORI4x128 %%GL, %%ZT13 + + ;; reduction + vmovdqa64 XWORD(%%ZT12), [rel POLY2] + VCLMUL_REDUCE %%HASH_IN_OUT, XWORD(%%ZT12), \ + XWORD(%%GH), XWORD(%%GL), XWORD(%%ZT13), XWORD(%%ZT14) + + vpxorq %%HASH_IN_OUT, %%HASH_IN_OUT, %%T7 +%endif + ;; The result is in %%HASH_IN_OUT + jmp %%_after_reduction +%endif + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;; After GHASH reduction +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +%%_small_initial_compute_done: + +%ifidn %%INSTANCE_TYPE, multi_call + ;; If using init/update/finalize, we need to xor any partial block data + ;; into the hash. +%if %%num_initial_blocks > 1 + ;; NOTE: for %%num_initial_blocks = 0 the xor never takes place +%if %%num_initial_blocks != 16 + ;; NOTE: for %%num_initial_blocks = 16, %%LENGTH, stored in [PBlockLen] is never zero + or %%LENGTH, %%LENGTH + je %%_after_reduction +%endif ; %%num_initial_blocks != 16 + vpxorq %%HASH_IN_OUT, %%HASH_IN_OUT, %%T7 +%endif ; %%num_initial_blocks > 1 +%endif ; %%INSTANCE_TYPE, multi_call + +%%_after_reduction: + ;; Final hash is now in HASH_IN_OUT + +%endmacro ; INITIAL_BLOCKS_PARTIAL + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;; Main GCM macro stitching cipher with GHASH +;;; - operates on single stream +;;; - encrypts 8 blocks at a time +;;; - ghash the 8 previously encrypted ciphertext blocks +;;; For partial block case and multi_call , AES_PARTIAL_BLOCK on output +;;; contains encrypted counter block. +%macro GHASH_8_ENCRYPT_8_PARALLEL 34-37 +%define %%GDATA %1 ; [in] key pointer +%define %%CYPH_PLAIN_OUT %2 ; [in] pointer to output buffer +%define %%PLAIN_CYPH_IN %3 ; [in] pointer to input buffer +%define %%DATA_OFFSET %4 ; [in] data offset +%define %%CTR1 %5 ; [in/out] ZMM counter blocks 0 to 3 +%define %%CTR2 %6 ; [in/out] ZMM counter blocks 4 to 7 +%define %%GHASHIN_AESOUT_B03 %7 ; [in/out] ZMM ghash in / aes out blocks 0 to 3 +%define %%GHASHIN_AESOUT_B47 %8 ; [in/out] ZMM ghash in / aes out blocks 4 to 7 +%define %%AES_PARTIAL_BLOCK %9 ; [out] XMM partial block (AES) +%define %%loop_idx %10 ; [in] counter block prep selection "add+shuffle" or "add" +%define %%ENC_DEC %11 ; [in] cipher direction +%define %%FULL_PARTIAL %12 ; [in] last block type selection "full" or "partial" +%define %%IA0 %13 ; [clobbered] temporary GP register +%define %%IA1 %14 ; [clobbered] temporary GP register +%define %%LENGTH %15 ; [in] length +%define %%INSTANCE_TYPE %16 ; [in] 'single_call' or 'multi_call' selection +%define %%GH4KEY %17 ; [in] ZMM with GHASH keys 4 to 1 +%define %%GH8KEY %18 ; [in] ZMM with GHASH keys 8 to 5 +%define %%SHFMSK %19 ; [in] ZMM with byte swap mask for pshufb +%define %%ZT1 %20 ; [clobbered] temporary ZMM (cipher) +%define %%ZT2 %21 ; [clobbered] temporary ZMM (cipher) +%define %%ZT3 %22 ; [clobbered] temporary ZMM (cipher) +%define %%ZT4 %23 ; [clobbered] temporary ZMM (cipher) +%define %%ZT5 %24 ; [clobbered] temporary ZMM (cipher) +%define %%ZT10 %25 ; [clobbered] temporary ZMM (ghash) +%define %%ZT11 %26 ; [clobbered] temporary ZMM (ghash) +%define %%ZT12 %27 ; [clobbered] temporary ZMM (ghash) +%define %%ZT13 %28 ; [clobbered] temporary ZMM (ghash) +%define %%ZT14 %29 ; [clobbered] temporary ZMM (ghash) +%define %%ZT15 %30 ; [clobbered] temporary ZMM (ghash) +%define %%ZT16 %31 ; [clobbered] temporary ZMM (ghash) +%define %%ZT17 %32 ; [clobbered] temporary ZMM (ghash) +%define %%MASKREG %33 ; [clobbered] mask register for partial loads/stores +%define %%DO_REDUCTION %34 ; [in] "reduction", "no_reduction", "final_reduction" +%define %%TO_REDUCE_L %35 ; [in/out] ZMM for low 4x128-bit in case of "no_reduction" +%define %%TO_REDUCE_H %36 ; [in/out] ZMM for hi 4x128-bit in case of "no_reduction" +%define %%TO_REDUCE_M %37 ; [in/out] ZMM for medium 4x128-bit in case of "no_reduction" + +%define %%GH1H %%ZT10 +%define %%GH1L %%ZT11 +%define %%GH1M1 %%ZT12 +%define %%GH1M2 %%ZT13 + +%define %%GH2H %%ZT14 +%define %%GH2L %%ZT15 +%define %%GH2M1 %%ZT16 +%define %%GH2M2 %%ZT17 + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; populate counter blocks for cipher part +%ifidn %%loop_idx, in_order + ;; %%CTR1 & %%CTR2 are shuffled outside the scope of this macro + ;; it has to be kept in unshuffled format + vpshufb %%ZT1, %%CTR1, %%SHFMSK + vpshufb %%ZT2, %%CTR2, %%SHFMSK +%else + vmovdqa64 %%ZT1, %%CTR1 + vmovdqa64 %%ZT2, %%CTR2 +%endif + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; stitch AES rounds with GHASH + +%assign aes_round 0 + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; AES round 0 - ARK + vbroadcastf64x2 %%ZT3, [%%GDATA + (aes_round * 16)] + ZMM_AESENC_ROUND_BLOCKS_0_16 %%ZT1, %%ZT2, no_zmm, no_zmm, \ + %%ZT3, aes_round, \ + %%ZT4, %%ZT5, no_zmm, no_zmm, \ + 8, NROUNDS +%assign aes_round (aes_round + 1) + + ;;================================================== + ;; GHASH 4 blocks + vpclmulqdq %%GH1H, %%GHASHIN_AESOUT_B47, %%GH4KEY, 0x11 ; a1*b1 + vpclmulqdq %%GH1L, %%GHASHIN_AESOUT_B47, %%GH4KEY, 0x00 ; a0*b0 + vpclmulqdq %%GH1M1, %%GHASHIN_AESOUT_B47, %%GH4KEY, 0x01 ; a1*b0 + vpclmulqdq %%GH1M2, %%GHASHIN_AESOUT_B47, %%GH4KEY, 0x10 ; a0*b1 + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; 3 AES rounds +%rep 3 + vbroadcastf64x2 %%ZT3, [%%GDATA + (aes_round * 16)] + ZMM_AESENC_ROUND_BLOCKS_0_16 %%ZT1, %%ZT2, no_zmm, no_zmm, \ + %%ZT3, aes_round, \ + %%ZT4, %%ZT5, no_zmm, no_zmm, \ + 8, NROUNDS +%assign aes_round (aes_round + 1) +%endrep ; 3 x AES ROUND + + ;; ================================================= + ;; GHASH 4 blocks + vpclmulqdq %%GH2M1, %%GHASHIN_AESOUT_B03, %%GH8KEY, 0x10 ; a0*b1 + vpclmulqdq %%GH2M2, %%GHASHIN_AESOUT_B03, %%GH8KEY, 0x01 ; a1*b0 + vpclmulqdq %%GH2H, %%GHASHIN_AESOUT_B03, %%GH8KEY, 0x11 ; a1*b1 + vpclmulqdq %%GH2L, %%GHASHIN_AESOUT_B03, %%GH8KEY, 0x00 ; a0*b0 + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; 3 AES rounds +%rep 3 + vbroadcastf64x2 %%ZT3, [%%GDATA + (aes_round * 16)] + ZMM_AESENC_ROUND_BLOCKS_0_16 %%ZT1, %%ZT2, no_zmm, no_zmm, \ + %%ZT3, aes_round, \ + %%ZT4, %%ZT5, no_zmm, no_zmm, \ + 8, NROUNDS +%assign aes_round (aes_round + 1) +%endrep ; 3 x AES ROUND + + ;; ================================================= + ;; gather GHASH in GH1L (low) and GH1H (high) +%ifidn %%DO_REDUCTION, no_reduction + vpternlogq %%GH1M1, %%GH1M2, %%GH2M1, 0x96 ; TM: GH1M1 ^= GH1M2 ^ GH2M1 + vpternlogq %%TO_REDUCE_M, %%GH1M1, %%GH2M2, 0x96 ; TM: TO_REDUCE_M ^= GH1M1 ^ GH2M2 + vpternlogq %%TO_REDUCE_H, %%GH1H, %%GH2H, 0x96 ; TH: TO_REDUCE_H ^= GH1H ^ GH2H + vpternlogq %%TO_REDUCE_L, %%GH1L, %%GH2L, 0x96 ; TL: TO_REDUCE_L ^= GH1L ^ GH2L +%endif +%ifidn %%DO_REDUCTION, do_reduction + ;; phase 1: add mid products together + vpternlogq %%GH1M1, %%GH1M2, %%GH2M1, 0x96 ; TM: GH1M1 ^= GH1M2 ^ GH2M1 + vpxorq %%GH1M1, %%GH1M1, %%GH2M2 + + vpsrldq %%GH2M1, %%GH1M1, 8 + vpslldq %%GH1M1, %%GH1M1, 8 +%endif +%ifidn %%DO_REDUCTION, final_reduction + ;; phase 1: add mid products together + vpternlogq %%GH1M1, %%GH1M2, %%GH2M1, 0x96 ; TM: GH1M1 ^= GH1M2 ^ GH2M1 + vpternlogq %%GH1M1, %%TO_REDUCE_M, %%GH2M2, 0x96 ; TM: GH1M1 ^= TO_REDUCE_M ^ GH2M2 + + vpsrldq %%GH2M1, %%GH1M1, 8 + vpslldq %%GH1M1, %%GH1M1, 8 +%endif + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; 2 AES rounds +%rep 2 + vbroadcastf64x2 %%ZT3, [%%GDATA + (aes_round * 16)] + ZMM_AESENC_ROUND_BLOCKS_0_16 %%ZT1, %%ZT2, no_zmm, no_zmm, \ + %%ZT3, aes_round, \ + %%ZT4, %%ZT5, no_zmm, no_zmm, \ + 8, NROUNDS +%assign aes_round (aes_round + 1) +%endrep ; 2 x AES ROUND + + ;; ================================================= + ;; Add mid product to high and low then + ;; horizontal xor of low and high 4x128 +%ifidn %%DO_REDUCTION, final_reduction + vpternlogq %%GH1H, %%GH2H, %%GH2M1, 0x96 ; TH = TH1 + TH2 + TM>>64 + vpxorq %%GH1H, %%TO_REDUCE_H + vpternlogq %%GH1L, %%GH2L, %%GH1M1, 0x96 ; TL = TL1 + TL2 + TM<<64 + vpxorq %%GH1L, %%TO_REDUCE_L +%endif +%ifidn %%DO_REDUCTION, do_reduction + vpternlogq %%GH1H, %%GH2H, %%GH2M1, 0x96 ; TH = TH1 + TH2 + TM>>64 + vpternlogq %%GH1L, %%GH2L, %%GH1M1, 0x96 ; TL = TL1 + TL2 + TM<<64 +%endif +%ifnidn %%DO_REDUCTION, no_reduction + VHPXORI4x128 %%GH1H, %%GH2H + VHPXORI4x128 %%GH1L, %%GH2L +%endif + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; 2 AES rounds +%rep 2 +%if (aes_round < (NROUNDS + 1)) + vbroadcastf64x2 %%ZT3, [%%GDATA + (aes_round * 16)] + ZMM_AESENC_ROUND_BLOCKS_0_16 %%ZT1, %%ZT2, no_zmm, no_zmm, \ + %%ZT3, aes_round, \ + %%ZT4, %%ZT5, no_zmm, no_zmm, \ + 8, NROUNDS +%assign aes_round (aes_round + 1) +%endif ; aes_round < (NROUNDS + 1) +%endrep + + ;; ================================================= + ;; first phase of reduction +%ifnidn %%DO_REDUCTION, no_reduction + vmovdqu64 XWORD(%%GH2M2), [rel POLY2] + vpclmulqdq XWORD(%%ZT15), XWORD(%%GH2M2), XWORD(%%GH1L), 0x01 + vpslldq XWORD(%%ZT15), XWORD(%%ZT15), 8 ; shift-L 2 DWs + vpxorq XWORD(%%ZT15), XWORD(%%GH1L), XWORD(%%ZT15) ; first phase of the reduct +%endif + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; 2 AES rounds +%rep 2 +%if (aes_round < (NROUNDS + 1)) + vbroadcastf64x2 %%ZT3, [%%GDATA + (aes_round * 16)] + ZMM_AESENC_ROUND_BLOCKS_0_16 %%ZT1, %%ZT2, no_zmm, no_zmm, \ + %%ZT3, aes_round, \ + %%ZT4, %%ZT5, no_zmm, no_zmm, \ + 8, NROUNDS +%assign aes_round (aes_round + 1) +%endif ; aes_round < (NROUNDS + 1) +%endrep + + ;; ================================================= + ;; second phase of the reduction +%ifnidn %%DO_REDUCTION, no_reduction + vpclmulqdq XWORD(%%ZT16), XWORD(%%GH2M2), XWORD(%%ZT15), 0x00 + vpsrldq XWORD(%%ZT16), XWORD(%%ZT16), 4 ; shift-R 1-DW to obtain 2-DWs shift-R + + vpclmulqdq XWORD(%%ZT13), XWORD(%%GH2M2), XWORD(%%ZT15), 0x10 + vpslldq XWORD(%%ZT13), XWORD(%%ZT13), 4 ; shift-L 1-DW for result without shifts + ;; ZT13 = ZT13 xor ZT16 xor GH1H + vpternlogq XWORD(%%ZT13), XWORD(%%ZT16), XWORD(%%GH1H), 0x96 +%endif + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; all remaining AES rounds but the last +%rep (NROUNDS + 2) +%if (aes_round < (NROUNDS + 1)) + vbroadcastf64x2 %%ZT3, [%%GDATA + (aes_round * 16)] + ZMM_AESENC_ROUND_BLOCKS_0_16 %%ZT1, %%ZT2, no_zmm, no_zmm, \ + %%ZT3, aes_round, \ + %%ZT4, %%ZT5, no_zmm, no_zmm, \ + 8, NROUNDS +%assign aes_round (aes_round + 1) +%endif ; aes_round < (NROUNDS + 1) +%endrep + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; load/store mask (partial case) and load the text data +%ifidn %%FULL_PARTIAL, full + VX512LDR %%ZT4, [%%PLAIN_CYPH_IN + %%DATA_OFFSET] + VX512LDR %%ZT5, [%%PLAIN_CYPH_IN + %%DATA_OFFSET + 64] +%else + lea %%IA0, [rel byte64_len_to_mask_table] + mov %%IA1, %%LENGTH + sub %%IA1, 64 + kmovq %%MASKREG, [%%IA0 + 8*%%IA1] + VX512LDR %%ZT4, [%%PLAIN_CYPH_IN + %%DATA_OFFSET] + vmovdqu8 %%ZT5{%%MASKREG}{z}, [%%PLAIN_CYPH_IN + %%DATA_OFFSET + 64] +%endif + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; the last AES round (NROUNDS + 1) and XOR against plain/cipher text + vbroadcastf64x2 %%ZT3, [%%GDATA + (aes_round * 16)] + ZMM_AESENC_ROUND_BLOCKS_0_16 %%ZT1, %%ZT2, no_zmm, no_zmm, \ + %%ZT3, aes_round, \ + %%ZT4, %%ZT5, no_zmm, no_zmm, \ + 8, NROUNDS + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; store the cipher/plain text data +%ifidn %%FULL_PARTIAL, full + VX512STR [%%CYPH_PLAIN_OUT + %%DATA_OFFSET], %%ZT1 + VX512STR [%%CYPH_PLAIN_OUT + %%DATA_OFFSET + 64], %%ZT2 +%else + VX512STR [%%CYPH_PLAIN_OUT + %%DATA_OFFSET], %%ZT1 + vmovdqu8 [%%CYPH_PLAIN_OUT + %%DATA_OFFSET + 64]{%%MASKREG}, %%ZT2 +%endif + + ;; ================================================= + ;; prep cipher text blocks for the next ghash round + +%ifnidn %%FULL_PARTIAL, full +%ifidn %%INSTANCE_TYPE, multi_call + ;; for partial block & multi_call we need encrypted counter block + vpxorq %%ZT3, %%ZT2, %%ZT5 + vextracti32x4 %%AES_PARTIAL_BLOCK, %%ZT3, 3 +%endif + ;; for GHASH computation purpose clear the top bytes of the partial block +%ifidn %%ENC_DEC, ENC + vmovdqu8 %%ZT2{%%MASKREG}{z}, %%ZT2 +%else + vmovdqu8 %%ZT5{%%MASKREG}{z}, %%ZT5 +%endif +%endif ; %ifnidn %%FULL_PARTIAL, full + + ;; ================================================= + ;; shuffle cipher text blocks for GHASH computation +%ifidn %%ENC_DEC, ENC + vpshufb %%GHASHIN_AESOUT_B03, %%ZT1, %%SHFMSK + vpshufb %%GHASHIN_AESOUT_B47, %%ZT2, %%SHFMSK +%else + vpshufb %%GHASHIN_AESOUT_B03, %%ZT4, %%SHFMSK + vpshufb %%GHASHIN_AESOUT_B47, %%ZT5, %%SHFMSK +%endif + +%ifidn %%DO_REDUCTION, do_reduction + ;; ================================================= + ;; XOR current GHASH value (ZT13) into block 0 + vpxorq %%GHASHIN_AESOUT_B03, %%ZT13 +%endif +%ifidn %%DO_REDUCTION, final_reduction + ;; ================================================= + ;; Return GHASH value (ZT13) in TO_REDUCE_L + vmovdqa64 %%TO_REDUCE_L, %%ZT13 +%endif + +%endmacro ; GHASH_8_ENCRYPT_8_PARALLEL + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;; Main GCM macro stitching cipher with GHASH +;;; - operates on single stream +;;; - encrypts 16 blocks at a time +;;; - ghash the 16 previously encrypted ciphertext blocks +;;; - no partial block or multi_call handling here +%macro GHASH_16_ENCRYPT_16_PARALLEL 42 +%define %%GDATA %1 ; [in] key pointer +%define %%CYPH_PLAIN_OUT %2 ; [in] pointer to output buffer +%define %%PLAIN_CYPH_IN %3 ; [in] pointer to input buffer +%define %%DATA_OFFSET %4 ; [in] data offset +%define %%CTR_BE %5 ; [in/out] ZMM counter blocks (last 4) in big-endian +%define %%CTR_CHECK %6 ; [in/out] GP with 8-bit counter for overflow check +%define %%HASHKEY_OFFSET %7 ; [in] numerical offset for the highest hash key +%define %%AESOUT_BLK_OFFSET %8 ; [in] numerical offset for AES-CTR out +%define %%GHASHIN_BLK_OFFSET %9 ; [in] numerical offset for GHASH blocks in +%define %%SHFMSK %10 ; [in] ZMM with byte swap mask for pshufb +%define %%ZT1 %11 ; [clobbered] temporary ZMM (cipher) +%define %%ZT2 %12 ; [clobbered] temporary ZMM (cipher) +%define %%ZT3 %13 ; [clobbered] temporary ZMM (cipher) +%define %%ZT4 %14 ; [clobbered] temporary ZMM (cipher) +%define %%ZT5 %15 ; [clobbered/out] temporary ZMM or GHASH OUT (final_reduction) +%define %%ZT6 %16 ; [clobbered] temporary ZMM (cipher) +%define %%ZT7 %17 ; [clobbered] temporary ZMM (cipher) +%define %%ZT8 %18 ; [clobbered] temporary ZMM (cipher) +%define %%ZT9 %19 ; [clobbered] temporary ZMM (cipher) +%define %%ZT10 %20 ; [clobbered] temporary ZMM (ghash) +%define %%ZT11 %21 ; [clobbered] temporary ZMM (ghash) +%define %%ZT12 %22 ; [clobbered] temporary ZMM (ghash) +%define %%ZT13 %23 ; [clobbered] temporary ZMM (ghash) +%define %%ZT14 %24 ; [clobbered] temporary ZMM (ghash) +%define %%ZT15 %25 ; [clobbered] temporary ZMM (ghash) +%define %%ZT16 %26 ; [clobbered] temporary ZMM (ghash) +%define %%ZT17 %27 ; [clobbered] temporary ZMM (ghash) +%define %%ZT18 %28 ; [clobbered] temporary ZMM (ghash) +%define %%ZT19 %29 ; [clobbered] temporary ZMM +%define %%ZT20 %30 ; [clobbered] temporary ZMM +%define %%ZT21 %31 ; [clobbered] temporary ZMM +%define %%ZT22 %32 ; [clobbered] temporary ZMM +%define %%ZT23 %33 ; [clobbered] temporary ZMM +%define %%ADDBE_4x4 %34 ; [in] ZMM with 4x128bits 4 in big-endian +%define %%ADDBE_1234 %35 ; [in] ZMM with 4x128bits 1, 2, 3 and 4 in big-endian +%define %%TO_REDUCE_L %36 ; [in/out] ZMM for low 4x128-bit GHASH sum +%define %%TO_REDUCE_H %37 ; [in/out] ZMM for hi 4x128-bit GHASH sum +%define %%TO_REDUCE_M %38 ; [in/out] ZMM for medium 4x128-bit GHASH sum +%define %%DO_REDUCTION %39 ; [in] "no_reduction", "final_reduction", "first_time" +%define %%ENC_DEC %40 ; [in] cipher direction +%define %%DATA_DISPL %41 ; [in] fixed numerical data displacement/offset +%define %%GHASH_IN %42 ; [in] current GHASH value or "no_ghash_in" + +%define %%B00_03 %%ZT1 +%define %%B04_07 %%ZT2 +%define %%B08_11 %%ZT3 +%define %%B12_15 %%ZT4 + +%define %%GH1H %%ZT5 ; @note: do not change this mapping +%define %%GH1L %%ZT6 +%define %%GH1M %%ZT7 +%define %%GH1T %%ZT8 + +%define %%GH2H %%ZT9 +%define %%GH2L %%ZT10 +%define %%GH2M %%ZT11 +%define %%GH2T %%ZT12 + +%define %%RED_POLY %%GH2T +%define %%RED_P1 %%GH2L +%define %%RED_T1 %%GH2H +%define %%RED_T2 %%GH2M + +%define %%GH3H %%ZT13 +%define %%GH3L %%ZT14 +%define %%GH3M %%ZT15 +%define %%GH3T %%ZT16 + +%define %%DATA1 %%ZT13 +%define %%DATA2 %%ZT14 +%define %%DATA3 %%ZT15 +%define %%DATA4 %%ZT16 + +%define %%AESKEY1 %%ZT17 +%define %%AESKEY2 %%ZT18 + +%define %%GHKEY1 %%ZT19 +%define %%GHKEY2 %%ZT20 +%define %%GHDAT1 %%ZT21 +%define %%GHDAT2 %%ZT22 + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; prepare counter blocks + + cmp BYTE(%%CTR_CHECK), (256 - 16) + jae %%_16_blocks_overflow + vpaddd %%B00_03, %%CTR_BE, %%ADDBE_1234 + vpaddd %%B04_07, %%B00_03, %%ADDBE_4x4 + vpaddd %%B08_11, %%B04_07, %%ADDBE_4x4 + vpaddd %%B12_15, %%B08_11, %%ADDBE_4x4 + jmp %%_16_blocks_ok +%%_16_blocks_overflow: + vpshufb %%CTR_BE, %%CTR_BE, %%SHFMSK + vmovdqa64 %%B12_15, [rel ddq_add_4444] + vpaddd %%B00_03, %%CTR_BE, [rel ddq_add_1234] + vpaddd %%B04_07, %%B00_03, %%B12_15 + vpaddd %%B08_11, %%B04_07, %%B12_15 + vpaddd %%B12_15, %%B08_11, %%B12_15 + vpshufb %%B00_03, %%SHFMSK + vpshufb %%B04_07, %%SHFMSK + vpshufb %%B08_11, %%SHFMSK + vpshufb %%B12_15, %%SHFMSK +%%_16_blocks_ok: + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; pre-load constants + vbroadcastf64x2 %%AESKEY1, [%%GDATA + (16 * 0)] +%ifnidn %%GHASH_IN, no_ghash_in + vpxorq %%GHDAT1, %%GHASH_IN, [rsp + %%GHASHIN_BLK_OFFSET + (0*64)] +%else + vmovdqa64 %%GHDAT1, [rsp + %%GHASHIN_BLK_OFFSET + (0*64)] +%endif + vmovdqu64 %%GHKEY1, [%%GDATA + %%HASHKEY_OFFSET + (0*64)] + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; save counter for the next round + ;; increment counter overflow check register + vshufi64x2 %%CTR_BE, %%B12_15, %%B12_15, 1111_1111b + add BYTE(%%CTR_CHECK), 16 + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; pre-load constants + vbroadcastf64x2 %%AESKEY2, [%%GDATA + (16 * 1)] + vmovdqu64 %%GHKEY2, [%%GDATA + %%HASHKEY_OFFSET + (1*64)] + vmovdqa64 %%GHDAT2, [rsp + %%GHASHIN_BLK_OFFSET + (1*64)] + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; stitch AES rounds with GHASH + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; AES round 0 - ARK + + vpxorq %%B00_03, %%AESKEY1 + vpxorq %%B04_07, %%AESKEY1 + vpxorq %%B08_11, %%AESKEY1 + vpxorq %%B12_15, %%AESKEY1 + vbroadcastf64x2 %%AESKEY1, [%%GDATA + (16 * 2)] + + ;;================================================== + ;; GHASH 4 blocks (15 to 12) + vpclmulqdq %%GH1H, %%GHDAT1, %%GHKEY1, 0x11 ; a1*b1 + vpclmulqdq %%GH1L, %%GHDAT1, %%GHKEY1, 0x00 ; a0*b0 + vpclmulqdq %%GH1M, %%GHDAT1, %%GHKEY1, 0x01 ; a1*b0 + vpclmulqdq %%GH1T, %%GHDAT1, %%GHKEY1, 0x10 ; a0*b1 + + vmovdqu64 %%GHKEY1, [%%GDATA + %%HASHKEY_OFFSET + (2*64)] + vmovdqa64 %%GHDAT1, [rsp + %%GHASHIN_BLK_OFFSET + (2*64)] + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; AES round 1 + vaesenc %%B00_03, %%B00_03, %%AESKEY2 + vaesenc %%B04_07, %%B04_07, %%AESKEY2 + vaesenc %%B08_11, %%B08_11, %%AESKEY2 + vaesenc %%B12_15, %%B12_15, %%AESKEY2 + vbroadcastf64x2 %%AESKEY2, [%%GDATA + (16 * 3)] + + ;; ================================================= + ;; GHASH 4 blocks (11 to 8) + vpclmulqdq %%GH2M, %%GHDAT2, %%GHKEY2, 0x10 ; a0*b1 + vpclmulqdq %%GH2T, %%GHDAT2, %%GHKEY2, 0x01 ; a1*b0 + vpclmulqdq %%GH2H, %%GHDAT2, %%GHKEY2, 0x11 ; a1*b1 + vpclmulqdq %%GH2L, %%GHDAT2, %%GHKEY2, 0x00 ; a0*b0 + + vmovdqu64 %%GHKEY2, [%%GDATA + %%HASHKEY_OFFSET + (3*64)] + vmovdqa64 %%GHDAT2, [rsp + %%GHASHIN_BLK_OFFSET + (3*64)] + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; AES round 2 + vaesenc %%B00_03, %%B00_03, %%AESKEY1 + vaesenc %%B04_07, %%B04_07, %%AESKEY1 + vaesenc %%B08_11, %%B08_11, %%AESKEY1 + vaesenc %%B12_15, %%B12_15, %%AESKEY1 + vbroadcastf64x2 %%AESKEY1, [%%GDATA + (16 * 4)] + + ;; ================================================= + ;; GHASH 4 blocks (7 to 4) + vpclmulqdq %%GH3M, %%GHDAT1, %%GHKEY1, 0x10 ; a0*b1 + vpclmulqdq %%GH3T, %%GHDAT1, %%GHKEY1, 0x01 ; a1*b0 + vpclmulqdq %%GH3H, %%GHDAT1, %%GHKEY1, 0x11 ; a1*b1 + vpclmulqdq %%GH3L, %%GHDAT1, %%GHKEY1, 0x00 ; a0*b0 + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; AES rounds 3 + vaesenc %%B00_03, %%B00_03, %%AESKEY2 + vaesenc %%B04_07, %%B04_07, %%AESKEY2 + vaesenc %%B08_11, %%B08_11, %%AESKEY2 + vaesenc %%B12_15, %%B12_15, %%AESKEY2 + vbroadcastf64x2 %%AESKEY2, [%%GDATA + (16 * 5)] + + ;; ================================================= + ;; Gather (XOR) GHASH for 12 blocks + vpternlogq %%GH1H, %%GH2H, %%GH3H, 0x96 + vpternlogq %%GH1L, %%GH2L, %%GH3L, 0x96 + vpternlogq %%GH1T, %%GH2T, %%GH3T, 0x96 + vpternlogq %%GH1M, %%GH2M, %%GH3M, 0x96 + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; AES rounds 4 + vaesenc %%B00_03, %%B00_03, %%AESKEY1 + vaesenc %%B04_07, %%B04_07, %%AESKEY1 + vaesenc %%B08_11, %%B08_11, %%AESKEY1 + vaesenc %%B12_15, %%B12_15, %%AESKEY1 + vbroadcastf64x2 %%AESKEY1, [%%GDATA + (16 * 6)] + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; load plain/cipher text (recycle GH3xx registers) + VX512LDR %%DATA1, [%%PLAIN_CYPH_IN + %%DATA_OFFSET + %%DATA_DISPL + (0 * 64)] + VX512LDR %%DATA2, [%%PLAIN_CYPH_IN + %%DATA_OFFSET + %%DATA_DISPL + (1 * 64)] + VX512LDR %%DATA3, [%%PLAIN_CYPH_IN + %%DATA_OFFSET + %%DATA_DISPL + (2 * 64)] + VX512LDR %%DATA4, [%%PLAIN_CYPH_IN + %%DATA_OFFSET + %%DATA_DISPL + (3 * 64)] + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; AES rounds 5 + vaesenc %%B00_03, %%B00_03, %%AESKEY2 + vaesenc %%B04_07, %%B04_07, %%AESKEY2 + vaesenc %%B08_11, %%B08_11, %%AESKEY2 + vaesenc %%B12_15, %%B12_15, %%AESKEY2 + vbroadcastf64x2 %%AESKEY2, [%%GDATA + (16 * 7)] + + ;; ================================================= + ;; GHASH 4 blocks (3 to 0) + vpclmulqdq %%GH2M, %%GHDAT2, %%GHKEY2, 0x10 ; a0*b1 + vpclmulqdq %%GH2T, %%GHDAT2, %%GHKEY2, 0x01 ; a1*b0 + vpclmulqdq %%GH2H, %%GHDAT2, %%GHKEY2, 0x11 ; a1*b1 + vpclmulqdq %%GH2L, %%GHDAT2, %%GHKEY2, 0x00 ; a0*b0 + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; AES round 6 + vaesenc %%B00_03, %%B00_03, %%AESKEY1 + vaesenc %%B04_07, %%B04_07, %%AESKEY1 + vaesenc %%B08_11, %%B08_11, %%AESKEY1 + vaesenc %%B12_15, %%B12_15, %%AESKEY1 + vbroadcastf64x2 %%AESKEY1, [%%GDATA + (16 * 8)] + + ;; ================================================= + ;; gather GHASH in GH1L (low) and GH1H (high) +%ifidn %%DO_REDUCTION, first_time + vpternlogq %%GH1M, %%GH1T, %%GH2T, 0x96 ; TM + vpxorq %%TO_REDUCE_M, %%GH1M, %%GH2M ; TM + vpxorq %%TO_REDUCE_H, %%GH1H, %%GH2H ; TH + vpxorq %%TO_REDUCE_L, %%GH1L, %%GH2L ; TL +%endif +%ifidn %%DO_REDUCTION, no_reduction + vpternlogq %%GH1M, %%GH1T, %%GH2T, 0x96 ; TM + vpternlogq %%TO_REDUCE_M, %%GH1M, %%GH2M, 0x96 ; TM + vpternlogq %%TO_REDUCE_H, %%GH1H, %%GH2H, 0x96 ; TH + vpternlogq %%TO_REDUCE_L, %%GH1L, %%GH2L, 0x96 ; TL +%endif +%ifidn %%DO_REDUCTION, final_reduction + ;; phase 1: add mid products together + ;; also load polynomial constant for reduction + vpternlogq %%GH1M, %%GH1T, %%GH2T, 0x96 ; TM + vpternlogq %%GH1M, %%TO_REDUCE_M, %%GH2M, 0x96 + + vpsrldq %%GH2M, %%GH1M, 8 + vpslldq %%GH1M, %%GH1M, 8 + + vmovdqa64 XWORD(%%RED_POLY), [rel POLY2] +%endif + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; AES round 7 + vaesenc %%B00_03, %%B00_03, %%AESKEY2 + vaesenc %%B04_07, %%B04_07, %%AESKEY2 + vaesenc %%B08_11, %%B08_11, %%AESKEY2 + vaesenc %%B12_15, %%B12_15, %%AESKEY2 + vbroadcastf64x2 %%AESKEY2, [%%GDATA + (16 * 9)] + + ;; ================================================= + ;; Add mid product to high and low +%ifidn %%DO_REDUCTION, final_reduction + vpternlogq %%GH1H, %%GH2H, %%GH2M, 0x96 ; TH = TH1 + TH2 + TM>>64 + vpxorq %%GH1H, %%TO_REDUCE_H + vpternlogq %%GH1L, %%GH2L, %%GH1M, 0x96 ; TL = TL1 + TL2 + TM<<64 + vpxorq %%GH1L, %%TO_REDUCE_L +%endif + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; AES round 8 + vaesenc %%B00_03, %%B00_03, %%AESKEY1 + vaesenc %%B04_07, %%B04_07, %%AESKEY1 + vaesenc %%B08_11, %%B08_11, %%AESKEY1 + vaesenc %%B12_15, %%B12_15, %%AESKEY1 + vbroadcastf64x2 %%AESKEY1, [%%GDATA + (16 * 10)] + + ;; ================================================= + ;; horizontal xor of low and high 4x128 +%ifidn %%DO_REDUCTION, final_reduction + VHPXORI4x128 %%GH1H, %%GH2H + VHPXORI4x128 %%GH1L, %%GH2L +%endif + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; AES round 9 + vaesenc %%B00_03, %%B00_03, %%AESKEY2 + vaesenc %%B04_07, %%B04_07, %%AESKEY2 + vaesenc %%B08_11, %%B08_11, %%AESKEY2 + vaesenc %%B12_15, %%B12_15, %%AESKEY2 +%if (NROUNDS >= 11) + vbroadcastf64x2 %%AESKEY2, [%%GDATA + (16 * 11)] +%endif + ;; ================================================= + ;; first phase of reduction +%ifidn %%DO_REDUCTION, final_reduction + vpclmulqdq XWORD(%%RED_P1), XWORD(%%RED_POLY), XWORD(%%GH1L), 0x01 + vpslldq XWORD(%%RED_P1), XWORD(%%RED_P1), 8 ; shift-L 2 DWs + vpxorq XWORD(%%RED_P1), XWORD(%%GH1L), XWORD(%%RED_P1) ; first phase of the reduct +%endif + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; AES rounds up to 11 (AES192) or 13 (AES256) + ;; AES128 is done +%if (NROUNDS >= 11) + vaesenc %%B00_03, %%B00_03, %%AESKEY1 + vaesenc %%B04_07, %%B04_07, %%AESKEY1 + vaesenc %%B08_11, %%B08_11, %%AESKEY1 + vaesenc %%B12_15, %%B12_15, %%AESKEY1 + vbroadcastf64x2 %%AESKEY1, [%%GDATA + (16 * 12)] + + vaesenc %%B00_03, %%B00_03, %%AESKEY2 + vaesenc %%B04_07, %%B04_07, %%AESKEY2 + vaesenc %%B08_11, %%B08_11, %%AESKEY2 + vaesenc %%B12_15, %%B12_15, %%AESKEY2 +%if (NROUNDS == 13) + vbroadcastf64x2 %%AESKEY2, [%%GDATA + (16 * 13)] + + vaesenc %%B00_03, %%B00_03, %%AESKEY1 + vaesenc %%B04_07, %%B04_07, %%AESKEY1 + vaesenc %%B08_11, %%B08_11, %%AESKEY1 + vaesenc %%B12_15, %%B12_15, %%AESKEY1 + vbroadcastf64x2 %%AESKEY1, [%%GDATA + (16 * 14)] + + vaesenc %%B00_03, %%B00_03, %%AESKEY2 + vaesenc %%B04_07, %%B04_07, %%AESKEY2 + vaesenc %%B08_11, %%B08_11, %%AESKEY2 + vaesenc %%B12_15, %%B12_15, %%AESKEY2 +%endif ; GCM256 / NROUNDS = 13 (15 including the first and the last) +%endif ; GCM192 / NROUNDS = 11 (13 including the first and the last) + + ;; ================================================= + ;; second phase of the reduction +%ifidn %%DO_REDUCTION, final_reduction + vpclmulqdq XWORD(%%RED_T1), XWORD(%%RED_POLY), XWORD(%%RED_P1), 0x00 + vpsrldq XWORD(%%RED_T1), XWORD(%%RED_T1), 4 ; shift-R 1-DW to obtain 2-DWs shift-R + + vpclmulqdq XWORD(%%RED_T2), XWORD(%%RED_POLY), XWORD(%%RED_P1), 0x10 + vpslldq XWORD(%%RED_T2), XWORD(%%RED_T2), 4 ; shift-L 1-DW for result without shifts + ;; GH1H = GH1H x RED_T1 x RED_T2 + vpternlogq XWORD(%%GH1H), XWORD(%%RED_T2), XWORD(%%RED_T1), 0x96 +%endif + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; the last AES round + vaesenclast %%B00_03, %%B00_03, %%AESKEY1 + vaesenclast %%B04_07, %%B04_07, %%AESKEY1 + vaesenclast %%B08_11, %%B08_11, %%AESKEY1 + vaesenclast %%B12_15, %%B12_15, %%AESKEY1 + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; XOR against plain/cipher text + vpxorq %%B00_03, %%B00_03, %%DATA1 + vpxorq %%B04_07, %%B04_07, %%DATA2 + vpxorq %%B08_11, %%B08_11, %%DATA3 + vpxorq %%B12_15, %%B12_15, %%DATA4 + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; store cipher/plain text + VX512STR [%%CYPH_PLAIN_OUT + %%DATA_OFFSET + %%DATA_DISPL + (0 * 64)], %%B00_03 + VX512STR [%%CYPH_PLAIN_OUT + %%DATA_OFFSET + %%DATA_DISPL + (1 * 64)], %%B04_07 + VX512STR [%%CYPH_PLAIN_OUT + %%DATA_OFFSET + %%DATA_DISPL + (2 * 64)], %%B08_11 + VX512STR [%%CYPH_PLAIN_OUT + %%DATA_OFFSET + %%DATA_DISPL + (3 * 64)], %%B12_15 + + ;; ================================================= + ;; shuffle cipher text blocks for GHASH computation +%ifidn %%ENC_DEC, ENC + vpshufb %%B00_03, %%B00_03, %%SHFMSK + vpshufb %%B04_07, %%B04_07, %%SHFMSK + vpshufb %%B08_11, %%B08_11, %%SHFMSK + vpshufb %%B12_15, %%B12_15, %%SHFMSK +%else + vpshufb %%B00_03, %%DATA1, %%SHFMSK + vpshufb %%B04_07, %%DATA2, %%SHFMSK + vpshufb %%B08_11, %%DATA3, %%SHFMSK + vpshufb %%B12_15, %%DATA4, %%SHFMSK +%endif + + ;; ================================================= + ;; store shuffled cipher text for ghashing + vmovdqa64 [rsp + %%AESOUT_BLK_OFFSET + (0*64)], %%B00_03 + vmovdqa64 [rsp + %%AESOUT_BLK_OFFSET + (1*64)], %%B04_07 + vmovdqa64 [rsp + %%AESOUT_BLK_OFFSET + (2*64)], %%B08_11 + vmovdqa64 [rsp + %%AESOUT_BLK_OFFSET + (3*64)], %%B12_15 + +%ifidn %%DO_REDUCTION, final_reduction + ;; ================================================= + ;; Return GHASH value through %%GH1H +%endif + +%endmacro ; GHASH_16_ENCRYPT_16_PARALLEL + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;; GHASH the last 8 ciphertext blocks. +;;; - optionally accepts GHASH product sums as input +%macro GHASH_LAST_8 10-13 +%define %%GDATA %1 ; [in] key pointer +%define %%BL47 %2 ; [in/clobbered] ZMM AES blocks 4 to 7 +%define %%BL03 %3 ; [in/cloberred] ZMM AES blocks 0 to 3 +%define %%ZTH %4 ; [cloberred] ZMM temporary +%define %%ZTM %5 ; [cloberred] ZMM temporary +%define %%ZTL %6 ; [cloberred] ZMM temporary +%define %%ZT01 %7 ; [cloberred] ZMM temporary +%define %%ZT02 %8 ; [cloberred] ZMM temporary +%define %%ZT03 %9 ; [cloberred] ZMM temporary +%define %%AAD_HASH %10 ; [out] XMM hash value +%define %%GH %11 ; [in/optional] ZMM with GHASH high product sum +%define %%GL %12 ; [in/optional] ZMM with GHASH low product sum +%define %%GM %13 ; [in/optional] ZMM with GHASH mid product sum + + VCLMUL_STEP1 %%GDATA, %%BL47, %%ZT01, %%ZTH, %%ZTM, %%ZTL + +%if %0 > 10 + ;; add optional sums before step2 + vpxorq %%ZTH, %%ZTH, %%GH + vpxorq %%ZTL, %%ZTL, %%GL + vpxorq %%ZTM, %%ZTM, %%GM +%endif + + VCLMUL_STEP2 %%GDATA, %%BL47, %%BL03, %%ZT01, %%ZT02, %%ZT03, %%ZTH, %%ZTM, %%ZTL + + vmovdqa64 XWORD(%%ZT03), [rel POLY2] + VCLMUL_REDUCE %%AAD_HASH, XWORD(%%ZT03), XWORD(%%BL47), XWORD(%%BL03), \ + XWORD(%%ZT01), XWORD(%%ZT02) +%endmacro ; GHASH_LAST_8 + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;; GHASH the last 7 cipher text blocks. +;;; - it uses same GHASH macros as GHASH_LAST_8 but with some twist +;;; - it loads GHASH keys for each of the data blocks, so that: +;;; - blocks 4, 5 and 6 will use GHASH keys 3, 2, 1 respectively +;;; - code ensures that unused block 7 and corresponding GHASH key are zeroed +;;; (clmul product is zero this way and will not affect the result) +;;; - blocks 0, 1, 2 and 3 will use USE GHASH keys 7, 6, 5 and 4 respectively +;;; - optionally accepts GHASH product sums as input +%macro GHASH_LAST_7 13-16 +%define %%GDATA %1 ; [in] key pointer +%define %%BL47 %2 ; [in/clobbered] ZMM AES blocks 4 to 7 +%define %%BL03 %3 ; [in/cloberred] ZMM AES blocks 0 to 3 +%define %%ZTH %4 ; [cloberred] ZMM temporary +%define %%ZTM %5 ; [cloberred] ZMM temporary +%define %%ZTL %6 ; [cloberred] ZMM temporary +%define %%ZT01 %7 ; [cloberred] ZMM temporary +%define %%ZT02 %8 ; [cloberred] ZMM temporary +%define %%ZT03 %9 ; [cloberred] ZMM temporary +%define %%ZT04 %10 ; [cloberred] ZMM temporary +%define %%AAD_HASH %11 ; [out] XMM hash value +%define %%MASKREG %12 ; [clobbered] mask register to use for loads +%define %%IA0 %13 ; [clobbered] GP temporary register +%define %%GH %14 ; [in/optional] ZMM with GHASH high product sum +%define %%GL %15 ; [in/optional] ZMM with GHASH low product sum +%define %%GM %16 ; [in/optional] ZMM with GHASH mid product sum + + vmovdqa64 XWORD(%%ZT04), [rel POLY2] + + VCLMUL_1_TO_8_STEP1 %%GDATA, %%BL47, %%ZT01, %%ZT02, %%ZTH, %%ZTM, %%ZTL, 7 + +%if %0 > 13 + ;; add optional sums before step2 + vpxorq %%ZTH, %%ZTH, %%GH + vpxorq %%ZTL, %%ZTL, %%GL + vpxorq %%ZTM, %%ZTM, %%GM +%endif + + VCLMUL_1_TO_8_STEP2 %%GDATA, %%BL47, %%BL03, \ + %%ZT01, %%ZT02, %%ZT03, \ + %%ZTH, %%ZTM, %%ZTL, 7 + + VCLMUL_REDUCE %%AAD_HASH, XWORD(%%ZT04), XWORD(%%BL47), XWORD(%%BL03), \ + XWORD(%%ZT01), XWORD(%%ZT02) +%endmacro ; GHASH_LAST_7 + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;; Encryption of a single block +%macro ENCRYPT_SINGLE_BLOCK 2 +%define %%GDATA %1 +%define %%XMM0 %2 + + vpxorq %%XMM0, %%XMM0, [%%GDATA+16*0] +%assign i 1 +%rep NROUNDS + vaesenc %%XMM0, [%%GDATA+16*i] +%assign i (i+1) +%endrep + vaesenclast %%XMM0, [%%GDATA+16*i] +%endmacro + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;; Save register content for the caller +%macro FUNC_SAVE 0 + ;; Required for Update/GMC_ENC + ;the number of pushes must equal STACK_OFFSET + mov rax, rsp + + sub rsp, STACK_FRAME_SIZE + and rsp, ~63 + + mov [rsp + STACK_GP_OFFSET + 0*8], r12 + mov [rsp + STACK_GP_OFFSET + 1*8], r13 + mov [rsp + STACK_GP_OFFSET + 2*8], r14 + mov [rsp + STACK_GP_OFFSET + 3*8], r15 + mov [rsp + STACK_GP_OFFSET + 4*8], rax ; stack + mov r14, rax ; r14 is used to retrieve stack args + mov [rsp + STACK_GP_OFFSET + 5*8], rbp + mov [rsp + STACK_GP_OFFSET + 6*8], rbx +%ifidn __OUTPUT_FORMAT__, win64 + mov [rsp + STACK_GP_OFFSET + 7*8], rdi + mov [rsp + STACK_GP_OFFSET + 8*8], rsi +%endif + +%ifidn __OUTPUT_FORMAT__, win64 + ; xmm6:xmm15 need to be maintained for Windows + vmovdqu [rsp + STACK_XMM_OFFSET + 0*16], xmm6 + vmovdqu [rsp + STACK_XMM_OFFSET + 1*16], xmm7 + vmovdqu [rsp + STACK_XMM_OFFSET + 2*16], xmm8 + vmovdqu [rsp + STACK_XMM_OFFSET + 3*16], xmm9 + vmovdqu [rsp + STACK_XMM_OFFSET + 4*16], xmm10 + vmovdqu [rsp + STACK_XMM_OFFSET + 5*16], xmm11 + vmovdqu [rsp + STACK_XMM_OFFSET + 6*16], xmm12 + vmovdqu [rsp + STACK_XMM_OFFSET + 7*16], xmm13 + vmovdqu [rsp + STACK_XMM_OFFSET + 8*16], xmm14 + vmovdqu [rsp + STACK_XMM_OFFSET + 9*16], xmm15 +%endif +%endmacro + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;; Restore register content for the caller +%macro FUNC_RESTORE 0 + +%ifdef SAFE_DATA + clear_scratch_gps_asm + clear_scratch_zmms_asm +%else + vzeroupper +%endif + +%ifidn __OUTPUT_FORMAT__, win64 + vmovdqu xmm15, [rsp + STACK_XMM_OFFSET + 9*16] + vmovdqu xmm14, [rsp + STACK_XMM_OFFSET + 8*16] + vmovdqu xmm13, [rsp + STACK_XMM_OFFSET + 7*16] + vmovdqu xmm12, [rsp + STACK_XMM_OFFSET + 6*16] + vmovdqu xmm11, [rsp + STACK_XMM_OFFSET + 5*16] + vmovdqu xmm10, [rsp + STACK_XMM_OFFSET + 4*16] + vmovdqu xmm9, [rsp + STACK_XMM_OFFSET + 3*16] + vmovdqu xmm8, [rsp + STACK_XMM_OFFSET + 2*16] + vmovdqu xmm7, [rsp + STACK_XMM_OFFSET + 1*16] + vmovdqu xmm6, [rsp + STACK_XMM_OFFSET + 0*16] +%endif + + ;; Required for Update/GMC_ENC + mov rbp, [rsp + STACK_GP_OFFSET + 5*8] + mov rbx, [rsp + STACK_GP_OFFSET + 6*8] +%ifidn __OUTPUT_FORMAT__, win64 + mov rdi, [rsp + STACK_GP_OFFSET + 7*8] + mov rsi, [rsp + STACK_GP_OFFSET + 8*8] +%endif + mov r12, [rsp + STACK_GP_OFFSET + 0*8] + mov r13, [rsp + STACK_GP_OFFSET + 1*8] + mov r14, [rsp + STACK_GP_OFFSET + 2*8] + mov r15, [rsp + STACK_GP_OFFSET + 3*8] + mov rsp, [rsp + STACK_GP_OFFSET + 4*8] ; stack +%endmacro + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;; GCM_INIT initializes a gcm_context_data struct to prepare for encoding/decoding. +;;; Input: gcm_key_data * (GDATA_KEY), gcm_context_data *(GDATA_CTX), IV, +;;; Additional Authentication data (A_IN), Additional Data length (A_LEN). +;;; Output: Updated GDATA_CTX with the hash of A_IN (AadHash) and initialized other parts of GDATA_CTX. +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +%macro GCM_INIT 21 +%define %%GDATA_KEY %1 ; [in] GCM expanded keys pointer +%define %%GDATA_CTX %2 ; [in] GCM context pointer +%define %%IV %3 ; [in] IV pointer +%define %%A_IN %4 ; [in] AAD pointer +%define %%A_LEN %5 ; [in] AAD length in bytes +%define %%GPR1 %6 ; [clobbered] GP register +%define %%GPR2 %7 ; [clobbered] GP register +%define %%GPR3 %8 ; [clobbered] GP register +%define %%MASKREG %9 ; [clobbered] mask register +%define %%AAD_HASH %10 ; [out] XMM for AAD_HASH value (xmm14) +%define %%CUR_COUNT %11 ; [out] XMM with current counter (xmm2) +%define %%ZT0 %12 ; [clobbered] ZMM register +%define %%ZT1 %13 ; [clobbered] ZMM register +%define %%ZT2 %14 ; [clobbered] ZMM register +%define %%ZT3 %15 ; [clobbered] ZMM register +%define %%ZT4 %16 ; [clobbered] ZMM register +%define %%ZT5 %17 ; [clobbered] ZMM register +%define %%ZT6 %18 ; [clobbered] ZMM register +%define %%ZT7 %19 ; [clobbered] ZMM register +%define %%ZT8 %20 ; [clobbered] ZMM register +%define %%ZT9 %21 ; [clobbered] ZMM register + + CALC_AAD_HASH %%A_IN, %%A_LEN, %%AAD_HASH, %%GDATA_KEY, \ + %%ZT0, %%ZT1, %%ZT2, %%ZT3, %%ZT4, %%ZT5, %%ZT6, %%ZT7, %%ZT8, %%ZT9, \ + %%GPR1, %%GPR2, %%GPR3, %%MASKREG + + mov %%GPR1, %%A_LEN + vmovdqu64 [%%GDATA_CTX + AadHash], %%AAD_HASH ; ctx.aad hash = aad_hash + mov [%%GDATA_CTX + AadLen], %%GPR1 ; ctx.aad_length = aad_length + + xor %%GPR1, %%GPR1 + mov [%%GDATA_CTX + InLen], %%GPR1 ; ctx.in_length = 0 + mov [%%GDATA_CTX + PBlockLen], %%GPR1 ; ctx.partial_block_length = 0 + + ;; read 12 IV bytes and pad with 0x00000001 + vmovdqu8 %%CUR_COUNT, [rel ONEf] + mov %%GPR2, %%IV + mov %%GPR1, 0x0000_0000_0000_0fff + kmovq %%MASKREG, %%GPR1 + vmovdqu8 %%CUR_COUNT{%%MASKREG}, [%%GPR2] ; ctr = IV | 0x1 + + vmovdqu64 [%%GDATA_CTX + OrigIV], %%CUR_COUNT ; ctx.orig_IV = iv + + ;; store IV as counter in LE format + vpshufb %%CUR_COUNT, [rel SHUF_MASK] + vmovdqu [%%GDATA_CTX + CurCount], %%CUR_COUNT ; ctx.current_counter = iv +%endmacro + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;; Cipher and ghash of payloads shorter than 256 bytes +;;; - number of blocks in the message comes as argument +;;; - depending on the number of blocks an optimized variant of +;;; INITIAL_BLOCKS_PARTIAL is invoked +%macro GCM_ENC_DEC_SMALL 42 +%define %%GDATA_KEY %1 ; [in] key pointer +%define %%GDATA_CTX %2 ; [in] context pointer +%define %%CYPH_PLAIN_OUT %3 ; [in] output buffer +%define %%PLAIN_CYPH_IN %4 ; [in] input buffer +%define %%PLAIN_CYPH_LEN %5 ; [in] buffer length +%define %%ENC_DEC %6 ; [in] cipher direction +%define %%DATA_OFFSET %7 ; [in] data offset +%define %%LENGTH %8 ; [in] data length +%define %%NUM_BLOCKS %9 ; [in] number of blocks to process 1 to 16 +%define %%CTR %10 ; [in/out] XMM counter block +%define %%HASH_IN_OUT %11 ; [in/out] XMM GHASH value +%define %%INSTANCE_TYPE %12 ; [in] single or multi call +%define %%ZTMP0 %13 ; [clobbered] ZMM register +%define %%ZTMP1 %14 ; [clobbered] ZMM register +%define %%ZTMP2 %15 ; [clobbered] ZMM register +%define %%ZTMP3 %16 ; [clobbered] ZMM register +%define %%ZTMP4 %17 ; [clobbered] ZMM register +%define %%ZTMP5 %18 ; [clobbered] ZMM register +%define %%ZTMP6 %19 ; [clobbered] ZMM register +%define %%ZTMP7 %20 ; [clobbered] ZMM register +%define %%ZTMP8 %21 ; [clobbered] ZMM register +%define %%ZTMP9 %22 ; [clobbered] ZMM register +%define %%ZTMP10 %23 ; [clobbered] ZMM register +%define %%ZTMP11 %24 ; [clobbered] ZMM register +%define %%ZTMP12 %25 ; [clobbered] ZMM register +%define %%ZTMP13 %26 ; [clobbered] ZMM register +%define %%ZTMP14 %27 ; [clobbered] ZMM register +%define %%ZTMP15 %28 ; [clobbered] ZMM register +%define %%ZTMP16 %29 ; [clobbered] ZMM register +%define %%ZTMP17 %30 ; [clobbered] ZMM register +%define %%ZTMP18 %31 ; [clobbered] ZMM register +%define %%ZTMP19 %32 ; [clobbered] ZMM register +%define %%ZTMP20 %33 ; [clobbered] ZMM register +%define %%ZTMP21 %34 ; [clobbered] ZMM register +%define %%ZTMP22 %35 ; [clobbered] ZMM register +%define %%GH %36 ; [in] ZMM ghash sum (high) +%define %%GL %37 ; [in] ZMM ghash sum (low) +%define %%GM %38 ; [in] ZMM ghash sum (middle) +%define %%IA0 %39 ; [clobbered] GP register +%define %%IA1 %40 ; [clobbered] GP register +%define %%MASKREG %41 ; [clobbered] mask register +%define %%SHUFMASK %42 ; [in] ZMM with BE/LE shuffle mask + + cmp %%NUM_BLOCKS, 8 + je %%_small_initial_num_blocks_is_8 + jl %%_small_initial_num_blocks_is_7_1 + + + cmp %%NUM_BLOCKS, 12 + je %%_small_initial_num_blocks_is_12 + jl %%_small_initial_num_blocks_is_11_9 + + ;; 16, 15, 14 or 13 + cmp %%NUM_BLOCKS, 16 + je %%_small_initial_num_blocks_is_16 + cmp %%NUM_BLOCKS, 15 + je %%_small_initial_num_blocks_is_15 + cmp %%NUM_BLOCKS, 14 + je %%_small_initial_num_blocks_is_14 + jmp %%_small_initial_num_blocks_is_13 + +%%_small_initial_num_blocks_is_11_9: + ;; 11, 10 or 9 + cmp %%NUM_BLOCKS, 11 + je %%_small_initial_num_blocks_is_11 + cmp %%NUM_BLOCKS, 10 + je %%_small_initial_num_blocks_is_10 + jmp %%_small_initial_num_blocks_is_9 + +%%_small_initial_num_blocks_is_7_1: + cmp %%NUM_BLOCKS, 4 + je %%_small_initial_num_blocks_is_4 + jl %%_small_initial_num_blocks_is_3_1 + ;; 7, 6 or 5 + cmp %%NUM_BLOCKS, 7 + je %%_small_initial_num_blocks_is_7 + cmp %%NUM_BLOCKS, 6 + je %%_small_initial_num_blocks_is_6 + jmp %%_small_initial_num_blocks_is_5 + +%%_small_initial_num_blocks_is_3_1: + ;; 3, 2 or 1 + cmp %%NUM_BLOCKS, 3 + je %%_small_initial_num_blocks_is_3 + cmp %%NUM_BLOCKS, 2 + je %%_small_initial_num_blocks_is_2 + + ;; for %%NUM_BLOCKS == 1, just fall through and no 'jmp' needed + + ;; Use rep to generate different block size variants + ;; - one block size has to be the first one +%assign num_blocks 1 +%rep 16 +%%_small_initial_num_blocks_is_ %+ num_blocks : + INITIAL_BLOCKS_PARTIAL %%GDATA_KEY, %%GDATA_CTX, %%CYPH_PLAIN_OUT, \ + %%PLAIN_CYPH_IN, %%LENGTH, %%DATA_OFFSET, num_blocks, \ + %%CTR, %%HASH_IN_OUT, %%ENC_DEC, %%INSTANCE_TYPE, \ + %%ZTMP0, %%ZTMP1, %%ZTMP2, %%ZTMP3, %%ZTMP4, \ + %%ZTMP5, %%ZTMP6, %%ZTMP7, %%ZTMP8, %%ZTMP9, \ + %%ZTMP10, %%ZTMP11, %%ZTMP12, %%ZTMP13, %%ZTMP14, \ + %%ZTMP15, %%ZTMP16, %%ZTMP17, %%ZTMP18, %%ZTMP19, \ + %%ZTMP20, %%ZTMP21, %%ZTMP22, \ + %%GH, %%GL, %%GM, \ + %%IA0, %%IA1, %%MASKREG, %%SHUFMASK +%if num_blocks != 16 + jmp %%_small_initial_blocks_encrypted +%endif +%assign num_blocks (num_blocks + 1) +%endrep + +%%_small_initial_blocks_encrypted: + +%endmacro ; GCM_ENC_DEC_SMALL + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; GCM_ENC_DEC Encodes/Decodes given data. Assumes that the passed gcm_context_data struct +; has been initialized by GCM_INIT +; Requires the input data be at least 1 byte long because of READ_SMALL_INPUT_DATA. +; Input: gcm_key_data struct* (GDATA_KEY), gcm_context_data *(GDATA_CTX), input text (PLAIN_CYPH_IN), +; input text length (PLAIN_CYPH_LEN) and whether encoding or decoding (ENC_DEC). +; Output: A cypher of the given plain text (CYPH_PLAIN_OUT), and updated GDATA_CTX +; Clobbers rax, r10-r15, and zmm0-zmm31, k1 +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +%macro GCM_ENC_DEC 7 +%define %%GDATA_KEY %1 ; [in] key pointer +%define %%GDATA_CTX %2 ; [in] context pointer +%define %%CYPH_PLAIN_OUT %3 ; [in] output buffer pointer +%define %%PLAIN_CYPH_IN %4 ; [in] input buffer pointer +%define %%PLAIN_CYPH_LEN %5 ; [in] buffer length +%define %%ENC_DEC %6 ; [in] cipher direction +%define %%INSTANCE_TYPE %7 ; [in] 'single_call' or 'multi_call' selection + +%define %%IA0 r10 +%define %%IA1 r12 +%define %%IA2 r13 +%define %%IA3 r15 +%define %%IA4 r11 +%define %%IA5 rax + +%define %%LENGTH %%IA2 +%define %%CTR_CHECK %%IA3 +%define %%DATA_OFFSET %%IA4 + +%define %%HASHK_PTR %%IA5 + +%define %%GCM_INIT_CTR_BLOCK xmm2 ; hardcoded in GCM_INIT for now + +%define %%AES_PARTIAL_BLOCK xmm8 +%define %%CTR_BLOCK2z zmm18 +%define %%CTR_BLOCKz zmm9 +%define %%CTR_BLOCKx xmm9 +%define %%AAD_HASHz zmm14 +%define %%AAD_HASHx xmm14 + +;;; ZTMP0 - ZTMP12 - used in by8 code, by128/48 code and GCM_ENC_DEC_SMALL +%define %%ZTMP0 zmm0 +%define %%ZTMP1 zmm3 +%define %%ZTMP2 zmm4 +%define %%ZTMP3 zmm5 +%define %%ZTMP4 zmm6 +%define %%ZTMP5 zmm7 +%define %%ZTMP6 zmm10 +%define %%ZTMP7 zmm11 +%define %%ZTMP8 zmm12 +%define %%ZTMP9 zmm13 +%define %%ZTMP10 zmm15 +%define %%ZTMP11 zmm16 +%define %%ZTMP12 zmm17 + +;;; ZTMP13 - ZTMP22 - used in by128/48 code and GCM_ENC_DEC_SMALL +;;; - some used by8 code as well through TMPxy names +%define %%ZTMP13 zmm19 +%define %%ZTMP14 zmm20 +%define %%ZTMP15 zmm21 +%define %%ZTMP16 zmm30 ; can be used in very/big_loop part +%define %%ZTMP17 zmm31 ; can be used in very/big_loop part +%define %%ZTMP18 zmm1 +%define %%ZTMP19 zmm2 +%define %%ZTMP20 zmm8 +%define %%ZTMP21 zmm22 +%define %%ZTMP22 zmm23 + +;;; Free to use: zmm24 - zmm29 +;;; - used by by128/48 and by8 +%define %%GH zmm24 +%define %%GL zmm25 +%define %%GM zmm26 +%define %%SHUF_MASK zmm29 +%define %%CTR_BLOCK_SAVE zmm28 + +;;; - used by by128/48 code only +%define %%ADDBE_4x4 zmm27 +%define %%ADDBE_1234 zmm28 ; conflicts with CTR_BLOCK_SAVE + +;; used by8 code only +%define %%GH4KEY %%ZTMP17 +%define %%GH8KEY %%ZTMP16 +%define %%BLK0 %%ZTMP18 +%define %%BLK1 %%ZTMP19 +%define %%ADD8BE zmm27 +%define %%ADD8LE %%ZTMP13 + +%define %%MASKREG k1 + +%ifdef GCM_BIG_DATA +;; reduction every 128 blocks, depth 32 blocks +;; @note 128 blocks is the maximum capacity of the stack frame when +;; GCM_BIG_DATA is defined +%assign very_big_loop_nblocks 128 +%assign very_big_loop_depth 32 +%endif + +;; reduction every 48 blocks, depth 32 blocks +;; @note 48 blocks is the maximum capacity of the stack frame when +;; GCM_BIG_DATA is not defined +%assign big_loop_nblocks 48 +%assign big_loop_depth 32 + +;;; Macro flow: +;;; - for message size bigger than very_big_loop_nblocks process data +;;; with "very_big_loop" parameters +;;; - for message size bigger than big_loop_nblocks process data +;;; with "big_loop" parameters +;;; - calculate the number of 16byte blocks in the message +;;; - process (number of 16byte blocks) mod 8 +;;; '%%_initial_num_blocks_is_# .. %%_initial_blocks_encrypted' +;;; - process 8 16 byte blocks at a time until all are done in %%_encrypt_by_8_new + +%ifidn __OUTPUT_FORMAT__, win64 + cmp %%PLAIN_CYPH_LEN, 0 +%else + or %%PLAIN_CYPH_LEN, %%PLAIN_CYPH_LEN +%endif + je %%_enc_dec_done + + xor %%DATA_OFFSET, %%DATA_OFFSET + + ;; Update length of data processed +%ifidn __OUTPUT_FORMAT__, win64 + mov %%IA0, %%PLAIN_CYPH_LEN + add [%%GDATA_CTX + InLen], %%IA0 +%else + add [%%GDATA_CTX + InLen], %%PLAIN_CYPH_LEN +%endif + vmovdqu64 %%AAD_HASHx, [%%GDATA_CTX + AadHash] + +%ifidn %%INSTANCE_TYPE, multi_call + ;; NOTE: partial block processing makes only sense for multi_call here. + ;; Used for the update flow - if there was a previous partial + ;; block fill the remaining bytes here. + PARTIAL_BLOCK %%GDATA_KEY, %%GDATA_CTX, %%CYPH_PLAIN_OUT, %%PLAIN_CYPH_IN, \ + %%PLAIN_CYPH_LEN, %%DATA_OFFSET, %%AAD_HASHx, %%ENC_DEC, \ + %%IA0, %%IA1, %%IA2, %%ZTMP0, %%ZTMP1, %%ZTMP2, %%ZTMP3, %%ZTMP4, \ + %%ZTMP5, %%ZTMP6, %%ZTMP7, %%ZTMP8, %%ZTMP9, %%MASKREG +%endif + + ;; lift counter block from GCM_INIT to here +%ifidn %%INSTANCE_TYPE, single_call + vmovdqu64 %%CTR_BLOCKx, %%GCM_INIT_CTR_BLOCK +%else + vmovdqu64 %%CTR_BLOCKx, [%%GDATA_CTX + CurCount] +%endif + + ;; Save the amount of data left to process in %%LENGTH + mov %%LENGTH, %%PLAIN_CYPH_LEN +%ifidn %%INSTANCE_TYPE, multi_call + ;; NOTE: %%DATA_OFFSET is zero in single_call case. + ;; Consequently PLAIN_CYPH_LEN will never be zero after + ;; %%DATA_OFFSET subtraction below. + ;; There may be no more data if it was consumed in the partial block. + sub %%LENGTH, %%DATA_OFFSET + je %%_enc_dec_done +%endif ; %%INSTANCE_TYPE, multi_call + + vmovdqa64 %%SHUF_MASK, [rel SHUF_MASK] + vmovdqa64 %%ADDBE_4x4, [rel ddq_addbe_4444] + +%ifdef GCM_BIG_DATA + vmovdqa64 %%ADDBE_1234, [rel ddq_addbe_1234] + + cmp %%LENGTH, (very_big_loop_nblocks * 16) + jl %%_message_below_very_big_nblocks + + INITIAL_BLOCKS_Nx16 %%PLAIN_CYPH_IN, %%CYPH_PLAIN_OUT, %%GDATA_KEY, %%DATA_OFFSET, \ + %%AAD_HASHz, %%CTR_BLOCKz, %%CTR_CHECK, \ + %%ZTMP0, %%ZTMP1, %%ZTMP2, %%ZTMP3, \ + %%ZTMP4, %%ZTMP5, %%ZTMP6, %%ZTMP7, \ + %%ZTMP8, %%ZTMP9, %%ZTMP10, %%ZTMP11, \ + %%ZTMP12, %%ZTMP13, %%ZTMP14, %%ZTMP15, \ + %%ZTMP16, %%ZTMP17, %%ZTMP18, %%ZTMP19, \ + %%ZTMP20, %%ZTMP21, %%ZTMP22, \ + %%GH, %%GL, %%GM, \ + %%ADDBE_4x4, %%ADDBE_1234, \ + %%SHUF_MASK, %%ENC_DEC, very_big_loop_nblocks, very_big_loop_depth + + sub %%LENGTH, (very_big_loop_nblocks * 16) + cmp %%LENGTH, (very_big_loop_nblocks * 16) + jl %%_no_more_very_big_nblocks + +%%_encrypt_very_big_nblocks: + GHASH_ENCRYPT_Nx16_PARALLEL \ + %%PLAIN_CYPH_IN, %%CYPH_PLAIN_OUT, %%GDATA_KEY, %%DATA_OFFSET, \ + %%CTR_BLOCKz, %%SHUF_MASK, \ + %%ZTMP0, %%ZTMP1, %%ZTMP2, %%ZTMP3, \ + %%ZTMP4, %%ZTMP5, %%ZTMP6, %%ZTMP7, \ + %%ZTMP8, %%ZTMP9, %%ZTMP10, %%ZTMP11, \ + %%ZTMP12, %%ZTMP13, %%ZTMP14, %%ZTMP15, \ + %%ZTMP16, %%ZTMP17, %%ZTMP18, %%ZTMP19, \ + %%ZTMP20, %%ZTMP21, %%ZTMP22, \ + %%GH, %%GL, %%GM, \ + %%ADDBE_4x4, %%ADDBE_1234, %%AAD_HASHz, \ + %%ENC_DEC, very_big_loop_nblocks, very_big_loop_depth, %%CTR_CHECK + + sub %%LENGTH, (very_big_loop_nblocks * 16) + cmp %%LENGTH, (very_big_loop_nblocks * 16) + jge %%_encrypt_very_big_nblocks + +%%_no_more_very_big_nblocks: + vpshufb %%CTR_BLOCKx, XWORD(%%SHUF_MASK) + vmovdqa64 XWORD(%%CTR_BLOCK_SAVE), %%CTR_BLOCKx + + GHASH_LAST_Nx16 %%GDATA_KEY, %%AAD_HASHz, \ + %%ZTMP0, %%ZTMP1, %%ZTMP2, %%ZTMP3, \ + %%ZTMP4, %%ZTMP5, %%ZTMP6, %%ZTMP7, \ + %%ZTMP8, %%ZTMP9, %%ZTMP10, %%ZTMP11, \ + %%ZTMP12, %%ZTMP13, %%ZTMP14, %%ZTMP15, \ + %%GH, %%GL, %%GM, very_big_loop_nblocks, very_big_loop_depth + + or %%LENGTH, %%LENGTH + jz %%_ghash_done + +%%_message_below_very_big_nblocks: +%endif ; GCM_BIG_DATA + + cmp %%LENGTH, (big_loop_nblocks * 16) + jl %%_message_below_big_nblocks + + ;; overwritten above by CTR_BLOCK_SAVE + vmovdqa64 %%ADDBE_1234, [rel ddq_addbe_1234] + + INITIAL_BLOCKS_Nx16 %%PLAIN_CYPH_IN, %%CYPH_PLAIN_OUT, %%GDATA_KEY, %%DATA_OFFSET, \ + %%AAD_HASHz, %%CTR_BLOCKz, %%CTR_CHECK, \ + %%ZTMP0, %%ZTMP1, %%ZTMP2, %%ZTMP3, \ + %%ZTMP4, %%ZTMP5, %%ZTMP6, %%ZTMP7, \ + %%ZTMP8, %%ZTMP9, %%ZTMP10, %%ZTMP11, \ + %%ZTMP12, %%ZTMP13, %%ZTMP14, %%ZTMP15, \ + %%ZTMP16, %%ZTMP17, %%ZTMP18, %%ZTMP19, \ + %%ZTMP20, %%ZTMP21, %%ZTMP22, \ + %%GH, %%GL, %%GM, \ + %%ADDBE_4x4, %%ADDBE_1234, \ + %%SHUF_MASK, %%ENC_DEC, big_loop_nblocks, big_loop_depth + + sub %%LENGTH, (big_loop_nblocks * 16) + cmp %%LENGTH, (big_loop_nblocks * 16) + jl %%_no_more_big_nblocks + +%%_encrypt_big_nblocks: + GHASH_ENCRYPT_Nx16_PARALLEL \ + %%PLAIN_CYPH_IN, %%CYPH_PLAIN_OUT, %%GDATA_KEY, %%DATA_OFFSET, \ + %%CTR_BLOCKz, %%SHUF_MASK, \ + %%ZTMP0, %%ZTMP1, %%ZTMP2, %%ZTMP3, \ + %%ZTMP4, %%ZTMP5, %%ZTMP6, %%ZTMP7, \ + %%ZTMP8, %%ZTMP9, %%ZTMP10, %%ZTMP11, \ + %%ZTMP12, %%ZTMP13, %%ZTMP14, %%ZTMP15, \ + %%ZTMP16, %%ZTMP17, %%ZTMP18, %%ZTMP19, \ + %%ZTMP20, %%ZTMP21, %%ZTMP22, \ + %%GH, %%GL, %%GM, \ + %%ADDBE_4x4, %%ADDBE_1234, %%AAD_HASHz, \ + %%ENC_DEC, big_loop_nblocks, big_loop_depth, %%CTR_CHECK + + sub %%LENGTH, (big_loop_nblocks * 16) + cmp %%LENGTH, (big_loop_nblocks * 16) + jge %%_encrypt_big_nblocks + +%%_no_more_big_nblocks: + vpshufb %%CTR_BLOCKx, XWORD(%%SHUF_MASK) + vmovdqa64 XWORD(%%CTR_BLOCK_SAVE), %%CTR_BLOCKx + + GHASH_LAST_Nx16 %%GDATA_KEY, %%AAD_HASHz, \ + %%ZTMP0, %%ZTMP1, %%ZTMP2, %%ZTMP3, \ + %%ZTMP4, %%ZTMP5, %%ZTMP6, %%ZTMP7, \ + %%ZTMP8, %%ZTMP9, %%ZTMP10, %%ZTMP11, \ + %%ZTMP12, %%ZTMP13, %%ZTMP14, %%ZTMP15, \ + %%GH, %%GL, %%GM, big_loop_nblocks, big_loop_depth + + or %%LENGTH, %%LENGTH + jz %%_ghash_done + +%%_message_below_big_nblocks: + + ;; Less than 256 bytes will be handled by the small message code, which + ;; can process up to 16 x blocks (16 bytes each) + cmp %%LENGTH, (16 * 16) + jge %%_large_message_path + + ;; Determine how many blocks to process + ;; - process one additional block if there is a partial block + mov %%IA1, %%LENGTH + add %%IA1, 15 + shr %%IA1, 4 + ;; %%IA1 can be in the range from 0 to 16 + + GCM_ENC_DEC_SMALL \ + %%GDATA_KEY, %%GDATA_CTX, %%CYPH_PLAIN_OUT, %%PLAIN_CYPH_IN, \ + %%PLAIN_CYPH_LEN, %%ENC_DEC, %%DATA_OFFSET, \ + %%LENGTH, %%IA1, %%CTR_BLOCKx, %%AAD_HASHx, %%INSTANCE_TYPE, \ + %%ZTMP0, %%ZTMP1, %%ZTMP2, %%ZTMP3, \ + %%ZTMP4, %%ZTMP5, %%ZTMP6, %%ZTMP7, \ + %%ZTMP8, %%ZTMP9, %%ZTMP10, %%ZTMP11, \ + %%ZTMP12, %%ZTMP13, %%ZTMP14, %%ZTMP15, \ + %%ZTMP16, %%ZTMP17, %%ZTMP18, %%ZTMP19, \ + %%ZTMP20, %%ZTMP21, %%ZTMP22, \ + no_zmm, no_zmm, no_zmm, \ + %%IA0, %%IA3, %%MASKREG, %%SHUF_MASK + + vmovdqa64 XWORD(%%CTR_BLOCK_SAVE), %%CTR_BLOCKx + + jmp %%_ghash_done + +%%_large_message_path: + ;; Determine how many blocks to process in INITIAL + ;; - process one additional block in INITIAL if there is a partial block + mov %%IA1, %%LENGTH + and %%IA1, 0xff + add %%IA1, 15 + shr %%IA1, 4 + ;; Don't allow 8 INITIAL blocks since this will + ;; be handled by the x8 partial loop. + and %%IA1, 7 + je %%_initial_num_blocks_is_0 + cmp %%IA1, 1 + je %%_initial_num_blocks_is_1 + cmp %%IA1, 2 + je %%_initial_num_blocks_is_2 + cmp %%IA1, 3 + je %%_initial_num_blocks_is_3 + cmp %%IA1, 4 + je %%_initial_num_blocks_is_4 + cmp %%IA1, 5 + je %%_initial_num_blocks_is_5 + cmp %%IA1, 6 + je %%_initial_num_blocks_is_6 + +%assign number_of_blocks 7 +%rep 8 +%%_initial_num_blocks_is_ %+ number_of_blocks: + INITIAL_BLOCKS %%GDATA_KEY, %%GDATA_CTX, %%CYPH_PLAIN_OUT, %%PLAIN_CYPH_IN, \ + %%LENGTH, %%DATA_OFFSET, number_of_blocks, %%CTR_BLOCKx, %%AAD_HASHz, \ + %%ZTMP0, %%ZTMP1, %%ZTMP2, %%ZTMP3, %%ZTMP4, \ + %%ZTMP5, %%ZTMP6, %%ZTMP7, %%ZTMP8, %%ZTMP9, %%ZTMP10, %%ZTMP11, \ + %%IA0, %%IA1, %%ENC_DEC, %%MASKREG, %%SHUF_MASK, no_partial_block +%if number_of_blocks != 0 + jmp %%_initial_blocks_encrypted +%endif +%assign number_of_blocks (number_of_blocks - 1) +%endrep + +%%_initial_blocks_encrypted: + vmovdqa64 XWORD(%%CTR_BLOCK_SAVE), %%CTR_BLOCKx + + ;; move cipher blocks from intial blocks to input of by8 macro + ;; and for GHASH_LAST_8/7 + ;; - ghash value already xor'ed into block 0 + vmovdqa64 %%BLK0, %%ZTMP0 + vmovdqa64 %%BLK1, %%ZTMP1 + + ;; The entire message cannot get processed in INITIAL_BLOCKS + ;; - GCM_ENC_DEC_SMALL handles up to 16 blocks + ;; - INITIAL_BLOCKS processes up to 15 blocks + ;; - no need to check for zero length at this stage + + ;; In order to have only one reduction at the end + ;; start HASH KEY pointer needs to be determined based on length and + ;; call type. + ;; - note that 8 blocks are already ciphered in INITIAL_BLOCKS and + ;; subtracted from LENGTH + lea %%IA1, [%%LENGTH + (8 * 16)] + add %%IA1, 15 + and %%IA1, 0x3f0 +%ifidn %%INSTANCE_TYPE, multi_call + ;; if partial block and multi_call then change hash key start by one + mov %%IA0, %%LENGTH + and %%IA0, 15 + add %%IA0, 15 + and %%IA0, 16 + sub %%IA1, %%IA0 +%endif + lea %%HASHK_PTR, [%%GDATA_KEY + HashKey + 16] + sub %%HASHK_PTR, %%IA1 + ;; HASHK_PTR + ;; - points at the first hash key to start GHASH with + ;; - needs to be updated as the message is processed (incremented) + + ;; pre-load constants + vmovdqa64 %%ADD8BE, [rel ddq_addbe_8888] + vmovdqa64 %%ADD8LE, [rel ddq_add_8888] + vpxorq %%GH, %%GH + vpxorq %%GL, %%GL + vpxorq %%GM, %%GM + + ;; prepare counter 8 blocks + vshufi64x2 %%CTR_BLOCKz, %%CTR_BLOCKz, %%CTR_BLOCKz, 0 + vpaddd %%CTR_BLOCK2z, %%CTR_BLOCKz, [rel ddq_add_5678] + vpaddd %%CTR_BLOCKz, %%CTR_BLOCKz, [rel ddq_add_1234] + vpshufb %%CTR_BLOCKz, %%SHUF_MASK + vpshufb %%CTR_BLOCK2z, %%SHUF_MASK + + ;; Process 7 full blocks plus a partial block + cmp %%LENGTH, 128 + jl %%_encrypt_by_8_partial + +%%_encrypt_by_8_parallel: + ;; in_order vs. out_order is an optimization to increment the counter + ;; without shuffling it back into little endian. + ;; %%CTR_CHECK keeps track of when we need to increment in order so + ;; that the carry is handled correctly. + + vmovq %%CTR_CHECK, XWORD(%%CTR_BLOCK_SAVE) + +%%_encrypt_by_8_new: + and WORD(%%CTR_CHECK), 255 + add WORD(%%CTR_CHECK), 8 + + vmovdqu64 %%GH4KEY, [%%HASHK_PTR + (4 * 16)] + vmovdqu64 %%GH8KEY, [%%HASHK_PTR + (0 * 16)] + + GHASH_8_ENCRYPT_8_PARALLEL %%GDATA_KEY, %%CYPH_PLAIN_OUT, %%PLAIN_CYPH_IN, \ + %%DATA_OFFSET, %%CTR_BLOCKz, %%CTR_BLOCK2z,\ + %%BLK0, %%BLK1, %%AES_PARTIAL_BLOCK, \ + out_order, %%ENC_DEC, full, %%IA0, %%IA1, %%LENGTH, %%INSTANCE_TYPE, \ + %%GH4KEY, %%GH8KEY, %%SHUF_MASK, \ + %%ZTMP0, %%ZTMP1, %%ZTMP2, %%ZTMP3, %%ZTMP4, %%ZTMP5, %%ZTMP6, \ + %%ZTMP7, %%ZTMP8, %%ZTMP9, %%ZTMP10, %%ZTMP11, %%ZTMP12, \ + %%MASKREG, no_reduction, %%GL, %%GH, %%GM + + add %%HASHK_PTR, (8 * 16) + add %%DATA_OFFSET, 128 + sub %%LENGTH, 128 + jz %%_encrypt_done + + cmp WORD(%%CTR_CHECK), (256 - 8) + jae %%_encrypt_by_8 + + vpaddd %%CTR_BLOCKz, %%ADD8BE + vpaddd %%CTR_BLOCK2z, %%ADD8BE + + cmp %%LENGTH, 128 + jl %%_encrypt_by_8_partial + + jmp %%_encrypt_by_8_new + +%%_encrypt_by_8: + vpshufb %%CTR_BLOCKz, %%SHUF_MASK + vpshufb %%CTR_BLOCK2z, %%SHUF_MASK + vpaddd %%CTR_BLOCKz, %%ADD8LE + vpaddd %%CTR_BLOCK2z, %%ADD8LE + vpshufb %%CTR_BLOCKz, %%SHUF_MASK + vpshufb %%CTR_BLOCK2z, %%SHUF_MASK + + cmp %%LENGTH, 128 + jge %%_encrypt_by_8_new + +%%_encrypt_by_8_partial: + ;; Test to see if we need a by 8 with partial block. At this point + ;; bytes remaining should be either zero or between 113-127. + ;; 'in_order' shuffle needed to align key for partial block xor. + ;; 'out_order' is a little faster because it avoids extra shuffles. + ;; - counter blocks for the next 8 blocks are prepared and in BE format + ;; - we can go ahead with out_order scenario + + vmovdqu64 %%GH4KEY, [%%HASHK_PTR + (4 * 16)] + vmovdqu64 %%GH8KEY, [%%HASHK_PTR + (0 * 16)] + + GHASH_8_ENCRYPT_8_PARALLEL %%GDATA_KEY, %%CYPH_PLAIN_OUT, %%PLAIN_CYPH_IN, \ + %%DATA_OFFSET, %%CTR_BLOCKz, %%CTR_BLOCK2z, \ + %%BLK0, %%BLK1, %%AES_PARTIAL_BLOCK, \ + out_order, %%ENC_DEC, partial, %%IA0, %%IA1, %%LENGTH, %%INSTANCE_TYPE, \ + %%GH4KEY, %%GH8KEY, %%SHUF_MASK, \ + %%ZTMP0, %%ZTMP1, %%ZTMP2, %%ZTMP3, %%ZTMP4, %%ZTMP5, %%ZTMP6, \ + %%ZTMP7, %%ZTMP8, %%ZTMP9, %%ZTMP10, %%ZTMP11, %%ZTMP12, \ + %%MASKREG, no_reduction, %%GL, %%GH, %%GM + + add %%HASHK_PTR, (8 * 16) + add %%DATA_OFFSET, (128 - 16) + sub %%LENGTH, (128 - 16) + +%ifidn %%INSTANCE_TYPE, multi_call + mov [%%GDATA_CTX + PBlockLen], %%LENGTH + vmovdqu64 [%%GDATA_CTX + PBlockEncKey], %%AES_PARTIAL_BLOCK +%endif + +%%_encrypt_done: + ;; Extract the last counter block in LE format + vextracti32x4 XWORD(%%CTR_BLOCK_SAVE), %%CTR_BLOCK2z, 3 + vpshufb XWORD(%%CTR_BLOCK_SAVE), XWORD(%%SHUF_MASK) + + ;; GHASH last cipher text blocks in xmm1-xmm8 + ;; - if block 8th is partial in a multi-call path then skip the block +%ifidn %%INSTANCE_TYPE, multi_call + cmp qword [%%GDATA_CTX + PBlockLen], 0 + jz %%_hash_last_8 + + ;; save the 8th partial block as GHASH_LAST_7 will clobber %%BLK1 + vextracti32x4 XWORD(%%ZTMP7), %%BLK1, 3 + + GHASH_LAST_7 %%GDATA_KEY, %%BLK1, %%BLK0, \ + %%ZTMP0, %%ZTMP1, %%ZTMP2, %%ZTMP3, %%ZTMP4, %%ZTMP5, %%ZTMP6, \ + %%AAD_HASHx, %%MASKREG, %%IA0, %%GH, %%GL, %%GM + + ;; XOR the partial word into the hash + vpxorq %%AAD_HASHx, %%AAD_HASHx, XWORD(%%ZTMP7) + jmp %%_ghash_done +%%_hash_last_8: +%endif + GHASH_LAST_8 %%GDATA_KEY, %%BLK1, %%BLK0, \ + %%ZTMP0, %%ZTMP1, %%ZTMP2, %%ZTMP3, %%ZTMP4, %%ZTMP5, %%AAD_HASHx, \ + %%GH, %%GL, %%GM +%%_ghash_done: + vmovdqu64 [%%GDATA_CTX + CurCount], XWORD(%%CTR_BLOCK_SAVE) + vmovdqu64 [%%GDATA_CTX + AadHash], %%AAD_HASHx +%%_enc_dec_done: + +%endmacro ; GCM_ENC_DEC + +;;; =========================================================================== +;;; =========================================================================== +;;; Encrypt/decrypt the initial 16 blocks +%macro INITIAL_BLOCKS_16 22 +%define %%IN %1 ; [in] input buffer +%define %%OUT %2 ; [in] output buffer +%define %%KP %3 ; [in] pointer to expanded keys +%define %%DATA_OFFSET %4 ; [in] data offset +%define %%GHASH %5 ; [in] ZMM with AAD (low 128 bits) +%define %%CTR %6 ; [in] ZMM with CTR BE blocks 4x128 bits +%define %%CTR_CHECK %7 ; [in/out] GPR with counter overflow check +%define %%ADDBE_4x4 %8 ; [in] ZMM 4x128bits with value 4 (big endian) +%define %%ADDBE_1234 %9 ; [in] ZMM 4x128bits with values 1, 2, 3 & 4 (big endian) +%define %%T0 %10 ; [clobered] temporary ZMM register +%define %%T1 %11 ; [clobered] temporary ZMM register +%define %%T2 %12 ; [clobered] temporary ZMM register +%define %%T3 %13 ; [clobered] temporary ZMM register +%define %%T4 %14 ; [clobered] temporary ZMM register +%define %%T5 %15 ; [clobered] temporary ZMM register +%define %%T6 %16 ; [clobered] temporary ZMM register +%define %%T7 %17 ; [clobered] temporary ZMM register +%define %%T8 %18 ; [clobered] temporary ZMM register +%define %%SHUF_MASK %19 ; [in] ZMM with BE/LE shuffle mask +%define %%ENC_DEC %20 ; [in] ENC (encrypt) or DEC (decrypt) selector +%define %%BLK_OFFSET %21 ; [in] stack frame offset to ciphered blocks +%define %%DATA_DISPL %22 ; [in] fixed numerical data displacement/offset + +%define %%B00_03 %%T5 +%define %%B04_07 %%T6 +%define %%B08_11 %%T7 +%define %%B12_15 %%T8 + +%assign stack_offset (%%BLK_OFFSET) + + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; prepare counter blocks + + cmp BYTE(%%CTR_CHECK), (256 - 16) + jae %%_next_16_overflow + vpaddd %%B00_03, %%CTR, %%ADDBE_1234 + vpaddd %%B04_07, %%B00_03, %%ADDBE_4x4 + vpaddd %%B08_11, %%B04_07, %%ADDBE_4x4 + vpaddd %%B12_15, %%B08_11, %%ADDBE_4x4 + jmp %%_next_16_ok +%%_next_16_overflow: + vpshufb %%CTR, %%CTR, %%SHUF_MASK + vmovdqa64 %%B12_15, [rel ddq_add_4444] + vpaddd %%B00_03, %%CTR, [rel ddq_add_1234] + vpaddd %%B04_07, %%B00_03, %%B12_15 + vpaddd %%B08_11, %%B04_07, %%B12_15 + vpaddd %%B12_15, %%B08_11, %%B12_15 + vpshufb %%B00_03, %%SHUF_MASK + vpshufb %%B04_07, %%SHUF_MASK + vpshufb %%B08_11, %%SHUF_MASK + vpshufb %%B12_15, %%SHUF_MASK +%%_next_16_ok: + vshufi64x2 %%CTR, %%B12_15, %%B12_15, 1111_1111b + add BYTE(%%CTR_CHECK), 16 + + ;; === load 16 blocks of data + VX512LDR %%T0, [%%IN + %%DATA_OFFSET + %%DATA_DISPL + (64*0)] + VX512LDR %%T1, [%%IN + %%DATA_OFFSET + %%DATA_DISPL + (64*1)] + VX512LDR %%T2, [%%IN + %%DATA_OFFSET + %%DATA_DISPL + (64*2)] + VX512LDR %%T3, [%%IN + %%DATA_OFFSET + %%DATA_DISPL + (64*3)] + + ;; move to AES encryption rounds +%assign i 0 + vbroadcastf64x2 %%T4, [%%KP + (16*i)] + vpxorq %%B00_03, %%B00_03, %%T4 + vpxorq %%B04_07, %%B04_07, %%T4 + vpxorq %%B08_11, %%B08_11, %%T4 + vpxorq %%B12_15, %%B12_15, %%T4 +%assign i (i + 1) + +%rep NROUNDS + vbroadcastf64x2 %%T4, [%%KP + (16*i)] + vaesenc %%B00_03, %%B00_03, %%T4 + vaesenc %%B04_07, %%B04_07, %%T4 + vaesenc %%B08_11, %%B08_11, %%T4 + vaesenc %%B12_15, %%B12_15, %%T4 +%assign i (i + 1) +%endrep + + vbroadcastf64x2 %%T4, [%%KP + (16*i)] + vaesenclast %%B00_03, %%B00_03, %%T4 + vaesenclast %%B04_07, %%B04_07, %%T4 + vaesenclast %%B08_11, %%B08_11, %%T4 + vaesenclast %%B12_15, %%B12_15, %%T4 + + ;; xor against text + vpxorq %%B00_03, %%B00_03, %%T0 + vpxorq %%B04_07, %%B04_07, %%T1 + vpxorq %%B08_11, %%B08_11, %%T2 + vpxorq %%B12_15, %%B12_15, %%T3 + + ;; store + VX512STR [%%OUT + %%DATA_OFFSET + %%DATA_DISPL + (64*0)], %%B00_03 + VX512STR [%%OUT + %%DATA_OFFSET + %%DATA_DISPL + (64*1)], %%B04_07 + VX512STR [%%OUT + %%DATA_OFFSET + %%DATA_DISPL + (64*2)], %%B08_11 + VX512STR [%%OUT + %%DATA_OFFSET + %%DATA_DISPL + (64*3)], %%B12_15 + +%ifidn %%ENC_DEC, DEC + ;; decryption - cipher text needs to go to GHASH phase + vpshufb %%B00_03, %%T0, %%SHUF_MASK + vpshufb %%B04_07, %%T1, %%SHUF_MASK + vpshufb %%B08_11, %%T2, %%SHUF_MASK + vpshufb %%B12_15, %%T3, %%SHUF_MASK +%else + ;; encryption + vpshufb %%B00_03, %%B00_03, %%SHUF_MASK + vpshufb %%B04_07, %%B04_07, %%SHUF_MASK + vpshufb %%B08_11, %%B08_11, %%SHUF_MASK + vpshufb %%B12_15, %%B12_15, %%SHUF_MASK +%endif + +%ifnidn %%GHASH, no_ghash + ;; === xor cipher block 0 with GHASH for the next GHASH round + vpxorq %%B00_03, %%B00_03, %%GHASH +%endif + + vmovdqa64 [rsp + stack_offset + (0 * 64)], %%B00_03 + vmovdqa64 [rsp + stack_offset + (1 * 64)], %%B04_07 + vmovdqa64 [rsp + stack_offset + (2 * 64)], %%B08_11 + vmovdqa64 [rsp + stack_offset + (3 * 64)], %%B12_15 +%endmacro ;INITIAL_BLOCKS_16 + +;;; =========================================================================== +;;; =========================================================================== +;;; Encrypt the initial N x 16 blocks +;;; - A x 16 blocks are encrypted/decrypted first (pipeline depth) +;;; - B x 16 blocks are encrypted/decrypted and previous A x 16 are ghashed +;;; - A + B = N +%macro INITIAL_BLOCKS_Nx16 39 +%define %%IN %1 ; [in] input buffer +%define %%OUT %2 ; [in] output buffer +%define %%KP %3 ; [in] pointer to expanded keys +%define %%DATA_OFFSET %4 ; [in/out] data offset +%define %%GHASH %5 ; [in] ZMM with AAD (low 128 bits) +%define %%CTR %6 ; [in/out] ZMM with CTR: in - LE & 128b; out - BE & 4x128b +%define %%CTR_CHECK %7 ; [in/out] GPR with counter overflow check +%define %%T0 %8 ; [clobered] temporary ZMM register +%define %%T1 %9 ; [clobered] temporary ZMM register +%define %%T2 %10 ; [clobered] temporary ZMM register +%define %%T3 %11 ; [clobered] temporary ZMM register +%define %%T4 %12 ; [clobered] temporary ZMM register +%define %%T5 %13 ; [clobered] temporary ZMM register +%define %%T6 %14 ; [clobered] temporary ZMM register +%define %%T7 %15 ; [clobered] temporary ZMM register +%define %%T8 %16 ; [clobered] temporary ZMM register +%define %%T9 %17 ; [clobered] temporary ZMM register +%define %%T10 %18 ; [clobered] temporary ZMM register +%define %%T11 %19 ; [clobered] temporary ZMM register +%define %%T12 %20 ; [clobered] temporary ZMM register +%define %%T13 %21 ; [clobered] temporary ZMM register +%define %%T14 %22 ; [clobered] temporary ZMM register +%define %%T15 %23 ; [clobered] temporary ZMM register +%define %%T16 %24 ; [clobered] temporary ZMM register +%define %%T17 %25 ; [clobered] temporary ZMM register +%define %%T18 %26 ; [clobered] temporary ZMM register +%define %%T19 %27 ; [clobered] temporary ZMM register +%define %%T20 %28 ; [clobered] temporary ZMM register +%define %%T21 %29 ; [clobered] temporary ZMM register +%define %%T22 %30 ; [clobered] temporary ZMM register +%define %%GH %31 ; [out] ZMM ghash sum (high) +%define %%GL %32 ; [out] ZMM ghash sum (low) +%define %%GM %33 ; [out] ZMM ghash sum (middle) +%define %%ADDBE_4x4 %34 ; [in] ZMM 4x128bits with value 4 (big endian) +%define %%ADDBE_1234 %35 ; [in] ZMM 4x128bits with values 1, 2, 3 & 4 (big endian) +%define %%SHUF_MASK %36 ; [in] ZMM with BE/LE shuffle mask +%define %%ENC_DEC %37 ; [in] ENC (encrypt) or DEC (decrypt) selector +%define %%NBLOCKS %38 ; [in] number of blocks: multiple of 16 +%define %%DEPTH_BLK %39 ; [in] pipline depth, number of blocks (mulitple of 16) + +%assign aesout_offset (STACK_LOCAL_OFFSET + (0 * 16)) +%assign ghashin_offset (STACK_LOCAL_OFFSET + (0 * 16)) +%assign hkey_offset HashKey_ %+ %%NBLOCKS +%assign data_in_out_offset 0 + + ;; set up CTR_CHECK + vmovd DWORD(%%CTR_CHECK), XWORD(%%CTR) + and DWORD(%%CTR_CHECK), 255 + + ;; in LE format after init, convert to BE + vshufi64x2 %%CTR, %%CTR, %%CTR, 0 + vpshufb %%CTR, %%CTR, %%SHUF_MASK + + ;; ==== AES lead in + + ;; first 16 blocks - just cipher + INITIAL_BLOCKS_16 %%IN, %%OUT, %%KP, %%DATA_OFFSET, \ + %%GHASH, %%CTR, %%CTR_CHECK, %%ADDBE_4x4, %%ADDBE_1234, \ + %%T0, %%T1, %%T2, %%T3, %%T4, \ + %%T5, %%T6, %%T7, %%T8, \ + %%SHUF_MASK, %%ENC_DEC, aesout_offset, data_in_out_offset + +%assign aesout_offset (aesout_offset + (16 * 16)) +%assign data_in_out_offset (data_in_out_offset + (16 * 16)) + +%if (%%DEPTH_BLK > 16) +%rep ((%%DEPTH_BLK - 16) / 16) + INITIAL_BLOCKS_16 %%IN, %%OUT, %%KP, %%DATA_OFFSET, \ + no_ghash, %%CTR, %%CTR_CHECK, %%ADDBE_4x4, %%ADDBE_1234, \ + %%T0, %%T1, %%T2, %%T3, %%T4, \ + %%T5, %%T6, %%T7, %%T8, \ + %%SHUF_MASK, %%ENC_DEC, aesout_offset, data_in_out_offset +%assign aesout_offset (aesout_offset + (16 * 16)) +%assign data_in_out_offset (data_in_out_offset + (16 * 16)) +%endrep +%endif + + ;; ==== GHASH + AES follows + + ;; first 16 blocks stitched + GHASH_16_ENCRYPT_16_PARALLEL %%KP, %%OUT, %%IN, %%DATA_OFFSET, \ + %%CTR, %%CTR_CHECK, \ + hkey_offset, aesout_offset, ghashin_offset, %%SHUF_MASK, \ + %%T0, %%T1, %%T2, %%T3, \ + %%T4, %%T5, %%T6, %%T7, \ + %%T8, %%T9, %%T10, %%T11,\ + %%T12, %%T13, %%T14, %%T15,\ + %%T16, %%T17, %%T18, %%T19, \ + %%T20, %%T21, %%T22, \ + %%ADDBE_4x4, %%ADDBE_1234, \ + %%GL, %%GH, %%GM, \ + first_time, %%ENC_DEC, data_in_out_offset, no_ghash_in + +%if ((%%NBLOCKS - %%DEPTH_BLK) > 16) +%rep ((%%NBLOCKS - %%DEPTH_BLK - 16) / 16) +%assign ghashin_offset (ghashin_offset + (16 * 16)) +%assign hkey_offset (hkey_offset + (16 * 16)) +%assign aesout_offset (aesout_offset + (16 * 16)) +%assign data_in_out_offset (data_in_out_offset + (16 * 16)) + + ;; mid 16 blocks - stitched + GHASH_16_ENCRYPT_16_PARALLEL %%KP, %%OUT, %%IN, %%DATA_OFFSET, \ + %%CTR, %%CTR_CHECK, \ + hkey_offset, aesout_offset, ghashin_offset, %%SHUF_MASK, \ + %%T0, %%T1, %%T2, %%T3, \ + %%T4, %%T5, %%T6, %%T7, \ + %%T8, %%T9, %%T10, %%T11,\ + %%T12, %%T13, %%T14, %%T15,\ + %%T16, %%T17, %%T18, %%T19, \ + %%T20, %%T21, %%T22, \ + %%ADDBE_4x4, %%ADDBE_1234, \ + %%GL, %%GH, %%GM, \ + no_reduction, %%ENC_DEC, data_in_out_offset, no_ghash_in +%endrep +%endif + add %%DATA_OFFSET, (%%NBLOCKS * 16) + +%endmacro ;INITIAL_BLOCKS_Nx16 + +;;; =========================================================================== +;;; =========================================================================== +;;; GHASH the last 16 blocks of cipher text (last part of by 32/64/128 code) +%macro GHASH_LAST_Nx16 23 +%define %%KP %1 ; [in] pointer to expanded keys +%define %%GHASH %2 ; [out] ghash output +%define %%T1 %3 ; [clobbered] temporary ZMM +%define %%T2 %4 ; [clobbered] temporary ZMM +%define %%T3 %5 ; [clobbered] temporary ZMM +%define %%T4 %6 ; [clobbered] temporary ZMM +%define %%T5 %7 ; [clobbered] temporary ZMM +%define %%T6 %8 ; [clobbered] temporary ZMM +%define %%T7 %9 ; [clobbered] temporary ZMM +%define %%T8 %10 ; [clobbered] temporary ZMM +%define %%T9 %11 ; [clobbered] temporary ZMM +%define %%T10 %12 ; [clobbered] temporary ZMM +%define %%T11 %13 ; [clobbered] temporary ZMM +%define %%T12 %14 ; [clobbered] temporary ZMM +%define %%T13 %15 ; [clobbered] temporary ZMM +%define %%T14 %16 ; [clobbered] temporary ZMM +%define %%T15 %17 ; [clobbered] temporary ZMM +%define %%T16 %18 ; [clobbered] temporary ZMM +%define %%GH %19 ; [in/cloberred] ghash sum (high) +%define %%GL %20 ; [in/cloberred] ghash sum (low) +%define %%GM %21 ; [in/cloberred] ghash sum (medium) +%define %%LOOP_BLK %22 ; [in] numerical number of blocks handled by the loop +%define %%DEPTH_BLK %23 ; [in] numerical number, pipeline depth (ghash vs aes) + +%define %%T0H %%T1 +%define %%T0L %%T2 +%define %%T0M1 %%T3 +%define %%T0M2 %%T4 + +%define %%T1H %%T5 +%define %%T1L %%T6 +%define %%T1M1 %%T7 +%define %%T1M2 %%T8 + +%define %%T2H %%T9 +%define %%T2L %%T10 +%define %%T2M1 %%T11 +%define %%T2M2 %%T12 + +%define %%BLK1 %%T13 +%define %%BLK2 %%T14 + +%define %%HK1 %%T15 +%define %%HK2 %%T16 + +%assign hashk HashKey_ %+ %%DEPTH_BLK +%assign cipher_blk (STACK_LOCAL_OFFSET + ((%%LOOP_BLK - %%DEPTH_BLK) * 16)) + + ;; load cipher blocks and ghash keys + vmovdqa64 %%BLK1, [rsp + cipher_blk] + vmovdqa64 %%BLK2, [rsp + cipher_blk + 64] + vmovdqu64 %%HK1, [%%KP + hashk] + vmovdqu64 %%HK2, [%%KP + hashk + 64] + ;; ghash blocks 0-3 + vpclmulqdq %%T0H, %%BLK1, %%HK1, 0x11 ; %%TH = a1*b1 + vpclmulqdq %%T0L, %%BLK1, %%HK1, 0x00 ; %%TL = a0*b0 + vpclmulqdq %%T0M1, %%BLK1, %%HK1, 0x01 ; %%TM1 = a1*b0 + vpclmulqdq %%T0M2, %%BLK1, %%HK1, 0x10 ; %%TM2 = a0*b1 + ;; ghash blocks 4-7 + vpclmulqdq %%T1H, %%BLK2, %%HK2, 0x11 ; %%TTH = a1*b1 + vpclmulqdq %%T1L, %%BLK2, %%HK2, 0x00 ; %%TTL = a0*b0 + vpclmulqdq %%T1M1, %%BLK2, %%HK2, 0x01 ; %%TTM1 = a1*b0 + vpclmulqdq %%T1M2, %%BLK2, %%HK2, 0x10 ; %%TTM2 = a0*b1 + vpternlogq %%T0H, %%T1H, %%GH, 0x96 ; T0H = T0H + T1H + GH + vpternlogq %%T0L, %%T1L, %%GL, 0x96 ; T0L = T0L + T1L + GL + vpternlogq %%T0M1, %%T1M1, %%GM, 0x96 ; T0M1 = T0M1 + T1M1 + GM + vpxorq %%T0M2, %%T0M2, %%T1M2 ; T0M2 = T0M2 + T1M2 + +%rep ((%%DEPTH_BLK - 8) / 8) +%assign hashk (hashk + 128) +%assign cipher_blk (cipher_blk + 128) + + ;; remaining blocks + ;; load next 8 cipher blocks and corresponding ghash keys + vmovdqa64 %%BLK1, [rsp + cipher_blk] + vmovdqa64 %%BLK2, [rsp + cipher_blk + 64] + vmovdqu64 %%HK1, [%%KP + hashk] + vmovdqu64 %%HK2, [%%KP + hashk + 64] + ;; ghash blocks 0-3 + vpclmulqdq %%T1H, %%BLK1, %%HK1, 0x11 ; %%TH = a1*b1 + vpclmulqdq %%T1L, %%BLK1, %%HK1, 0x00 ; %%TL = a0*b0 + vpclmulqdq %%T1M1, %%BLK1, %%HK1, 0x01 ; %%TM1 = a1*b0 + vpclmulqdq %%T1M2, %%BLK1, %%HK1, 0x10 ; %%TM2 = a0*b1 + ;; ghash blocks 4-7 + vpclmulqdq %%T2H, %%BLK2, %%HK2, 0x11 ; %%TTH = a1*b1 + vpclmulqdq %%T2L, %%BLK2, %%HK2, 0x00 ; %%TTL = a0*b0 + vpclmulqdq %%T2M1, %%BLK2, %%HK2, 0x01 ; %%TTM1 = a1*b0 + vpclmulqdq %%T2M2, %%BLK2, %%HK2, 0x10 ; %%TTM2 = a0*b1 + ;; update sums + vpternlogq %%T0H, %%T1H, %%T2H, 0x96 ; TH = T0H + T1H + T2H + vpternlogq %%T0L, %%T1L, %%T2L, 0x96 ; TL = T0L + T1L + T2L + vpternlogq %%T0M1, %%T1M1, %%T2M1, 0x96 ; TM1 = T0M1 + T1M1 xor T2M1 + vpternlogq %%T0M2, %%T1M2, %%T2M2, 0x96 ; TM2 = T0M2 + T1M1 xor T2M2 +%endrep + + ;; integrate TM into TH and TL + vpxorq %%T0M1, %%T0M1, %%T0M2 + vpsrldq %%T1M1, %%T0M1, 8 + vpslldq %%T1M2, %%T0M1, 8 + vpxorq %%T0H, %%T0H, %%T1M1 + vpxorq %%T0L, %%T0L, %%T1M2 + + ;; add TH and TL 128-bit words horizontally + VHPXORI4x128 %%T0H, %%T2M1 + VHPXORI4x128 %%T0L, %%T2M2 + + ;; reduction + vmovdqa64 %%HK1, [rel POLY2] + VCLMUL_REDUCE %%GHASH, %%HK1, %%T0H, %%T0L, %%T0M1, %%T0M2 +%endmacro + +;;; =========================================================================== +;;; =========================================================================== +;;; Encrypt & ghash multiples of 16 blocks + +%macro GHASH_ENCRYPT_Nx16_PARALLEL 39 +%define %%IN %1 ; [in] input buffer +%define %%OUT %2 ; [in] output buffer +%define %%GDATA_KEY %3 ; [in] pointer to expanded keys +%define %%DATA_OFFSET %4 ; [in/out] data offset +%define %%CTR_BE %5 ; [in/out] ZMM last counter block +%define %%SHFMSK %6 ; [in] ZMM with byte swap mask for pshufb +%define %%ZT0 %7 ; [clobered] temporary ZMM register +%define %%ZT1 %8 ; [clobered] temporary ZMM register +%define %%ZT2 %9 ; [clobered] temporary ZMM register +%define %%ZT3 %10 ; [clobered] temporary ZMM register +%define %%ZT4 %11 ; [clobered] temporary ZMM register +%define %%ZT5 %12 ; [clobered] temporary ZMM register +%define %%ZT6 %13 ; [clobered] temporary ZMM register +%define %%ZT7 %14 ; [clobered] temporary ZMM register +%define %%ZT8 %15 ; [clobered] temporary ZMM register +%define %%ZT9 %16 ; [clobered] temporary ZMM register +%define %%ZT10 %17 ; [clobered] temporary ZMM register +%define %%ZT11 %18 ; [clobered] temporary ZMM register +%define %%ZT12 %19 ; [clobered] temporary ZMM register +%define %%ZT13 %20 ; [clobered] temporary ZMM register +%define %%ZT14 %21 ; [clobered] temporary ZMM register +%define %%ZT15 %22 ; [clobered] temporary ZMM register +%define %%ZT16 %23 ; [clobered] temporary ZMM register +%define %%ZT17 %24 ; [clobered] temporary ZMM register +%define %%ZT18 %25 ; [clobered] temporary ZMM register +%define %%ZT19 %26 ; [clobered] temporary ZMM register +%define %%ZT20 %27 ; [clobered] temporary ZMM register +%define %%ZT21 %28 ; [clobered] temporary ZMM register +%define %%ZT22 %29 ; [clobered] temporary ZMM register +%define %%GTH %30 ; [in/out] ZMM GHASH sum (high) +%define %%GTL %31 ; [in/out] ZMM GHASH sum (low) +%define %%GTM %32 ; [in/out] ZMM GHASH sum (medium) +%define %%ADDBE_4x4 %33 ; [in] ZMM 4x128bits with value 4 (big endian) +%define %%ADDBE_1234 %34 ; [in] ZMM 4x128bits with values 1, 2, 3 & 4 (big endian) +%define %%GHASH %35 ; [clobbered] ZMM with intermidiate GHASH value +%define %%ENC_DEC %36 ; [in] ENC (encrypt) or DEC (decrypt) selector +%define %%NUM_BLOCKS %37 ; [in] number of blocks to process in the loop +%define %%DEPTH_BLK %38 ; [in] pipeline depth in blocks +%define %%CTR_CHECK %39 ; [in/out] counter to check byte overflow + +%assign aesout_offset (STACK_LOCAL_OFFSET + (0 * 16)) +%assign ghashin_offset (STACK_LOCAL_OFFSET + ((%%NUM_BLOCKS - %%DEPTH_BLK) * 16)) +%assign hkey_offset HashKey_ %+ %%DEPTH_BLK +%assign data_in_out_offset 0 + + ;; mid 16 blocks +%if (%%DEPTH_BLK > 16) +%rep ((%%DEPTH_BLK - 16) / 16) + GHASH_16_ENCRYPT_16_PARALLEL %%GDATA_KEY, %%OUT, %%IN, %%DATA_OFFSET, \ + %%CTR_BE, %%CTR_CHECK, \ + hkey_offset, aesout_offset, ghashin_offset, %%SHFMSK, \ + %%ZT0, %%ZT1, %%ZT2, %%ZT3, \ + %%ZT4, %%ZT5, %%ZT6, %%ZT7, \ + %%ZT8, %%ZT9, %%ZT10, %%ZT11,\ + %%ZT12, %%ZT13, %%ZT14, %%ZT15,\ + %%ZT16, %%ZT17, %%ZT18, %%ZT19, \ + %%ZT20, %%ZT21, %%ZT22, \ + %%ADDBE_4x4, %%ADDBE_1234, \ + %%GTL, %%GTH, %%GTM, \ + no_reduction, %%ENC_DEC, data_in_out_offset, no_ghash_in + +%assign aesout_offset (aesout_offset + (16 * 16)) +%assign ghashin_offset (ghashin_offset + (16 * 16)) +%assign hkey_offset (hkey_offset + (16 * 16)) +%assign data_in_out_offset (data_in_out_offset + (16 * 16)) +%endrep +%endif + + ;; 16 blocks with reduction + GHASH_16_ENCRYPT_16_PARALLEL %%GDATA_KEY, %%OUT, %%IN, %%DATA_OFFSET, \ + %%CTR_BE, %%CTR_CHECK, \ + HashKey_16, aesout_offset, ghashin_offset, %%SHFMSK, \ + %%ZT0, %%ZT1, %%ZT2, %%ZT3, \ + %%ZT4, %%ZT5, %%ZT6, %%ZT7, \ + %%ZT8, %%ZT9, %%ZT10, %%ZT11,\ + %%ZT12, %%ZT13, %%ZT14, %%ZT15,\ + %%ZT16, %%ZT17, %%ZT18, %%ZT19, \ + %%ZT20, %%ZT21, %%ZT22, \ + %%ADDBE_4x4, %%ADDBE_1234, \ + %%GTL, %%GTH, %%GTM, \ + final_reduction, %%ENC_DEC, data_in_out_offset, no_ghash_in + +%assign aesout_offset (aesout_offset + (16 * 16)) +%assign data_in_out_offset (data_in_out_offset + (16 * 16)) +%assign ghashin_offset (STACK_LOCAL_OFFSET + (0 * 16)) +%assign hkey_offset HashKey_ %+ %%NUM_BLOCKS + + ;; === xor cipher block 0 with GHASH (ZT4) + vmovdqa64 %%GHASH, %%ZT4 + + ;; start the pipeline again + GHASH_16_ENCRYPT_16_PARALLEL %%GDATA_KEY, %%OUT, %%IN, %%DATA_OFFSET, \ + %%CTR_BE, %%CTR_CHECK, \ + hkey_offset, aesout_offset, ghashin_offset, %%SHFMSK, \ + %%ZT0, %%ZT1, %%ZT2, %%ZT3, \ + %%ZT4, %%ZT5, %%ZT6, %%ZT7, \ + %%ZT8, %%ZT9, %%ZT10, %%ZT11,\ + %%ZT12, %%ZT13, %%ZT14, %%ZT15,\ + %%ZT16, %%ZT17, %%ZT18, %%ZT19, \ + %%ZT20, %%ZT21, %%ZT22, \ + %%ADDBE_4x4, %%ADDBE_1234, \ + %%GTL, %%GTH, %%GTM, \ + first_time, %%ENC_DEC, data_in_out_offset, %%GHASH + +%if ((%%NUM_BLOCKS - %%DEPTH_BLK) > 16) +%rep ((%%NUM_BLOCKS - %%DEPTH_BLK - 16 ) / 16) + +%assign aesout_offset (aesout_offset + (16 * 16)) +%assign data_in_out_offset (data_in_out_offset + (16 * 16)) +%assign ghashin_offset (ghashin_offset + (16 * 16)) +%assign hkey_offset (hkey_offset + (16 * 16)) + + GHASH_16_ENCRYPT_16_PARALLEL %%GDATA_KEY, %%OUT, %%IN, %%DATA_OFFSET, \ + %%CTR_BE, %%CTR_CHECK, \ + hkey_offset, aesout_offset, ghashin_offset, %%SHFMSK, \ + %%ZT0, %%ZT1, %%ZT2, %%ZT3, \ + %%ZT4, %%ZT5, %%ZT6, %%ZT7, \ + %%ZT8, %%ZT9, %%ZT10, %%ZT11,\ + %%ZT12, %%ZT13, %%ZT14, %%ZT15,\ + %%ZT16, %%ZT17, %%ZT18, %%ZT19, \ + %%ZT20, %%ZT21, %%ZT22, \ + %%ADDBE_4x4, %%ADDBE_1234, \ + %%GTL, %%GTH, %%GTM, \ + no_reduction, %%ENC_DEC, data_in_out_offset, no_ghash_in +%endrep +%endif + + add %%DATA_OFFSET, (%%NUM_BLOCKS * 16) + +%endmacro ;GHASH_ENCRYPT_Nx16_PARALLEL +;;; =========================================================================== + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; GCM_COMPLETE Finishes Encyrption/Decryption of last partial block after GCM_UPDATE finishes. +; Input: A gcm_key_data * (GDATA_KEY), gcm_context_data (GDATA_CTX) and whether encoding or decoding (ENC_DEC). +; Output: Authorization Tag (AUTH_TAG) and Authorization Tag length (AUTH_TAG_LEN) +; Clobbers rax, r10-r12, and xmm0, xmm1, xmm5, xmm6, xmm9, xmm11, xmm14, xmm15 +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +%macro GCM_COMPLETE 6 +%define %%GDATA_KEY %1 +%define %%GDATA_CTX %2 +%define %%AUTH_TAG %3 +%define %%AUTH_TAG_LEN %4 +%define %%ENC_DEC %5 +%define %%INSTANCE_TYPE %6 +%define %%PLAIN_CYPH_LEN rax + + vmovdqu xmm13, [%%GDATA_KEY + HashKey] + ;; Start AES as early as possible + vmovdqu xmm9, [%%GDATA_CTX + OrigIV] ; xmm9 = Y0 + ENCRYPT_SINGLE_BLOCK %%GDATA_KEY, xmm9 ; E(K, Y0) + +%ifidn %%INSTANCE_TYPE, multi_call + ;; If the GCM function is called as a single function call rather + ;; than invoking the individual parts (init, update, finalize) we + ;; can remove a write to read dependency on AadHash. + vmovdqu xmm14, [%%GDATA_CTX + AadHash] + + ;; Encrypt the final partial block. If we did this as a single call then + ;; the partial block was handled in the main GCM_ENC_DEC macro. + mov r12, [%%GDATA_CTX + PBlockLen] + cmp r12, 0 + + je %%_partial_done + + GHASH_MUL xmm14, xmm13, xmm0, xmm10, xmm11, xmm5, xmm6 ;GHASH computation for the last <16 Byte block + vmovdqu [%%GDATA_CTX + AadHash], xmm14 + +%%_partial_done: + +%endif + + mov r12, [%%GDATA_CTX + AadLen] ; r12 = aadLen (number of bytes) + mov %%PLAIN_CYPH_LEN, [%%GDATA_CTX + InLen] + + shl r12, 3 ; convert into number of bits + vmovd xmm15, r12d ; len(A) in xmm15 + + shl %%PLAIN_CYPH_LEN, 3 ; len(C) in bits (*128) + vmovq xmm1, %%PLAIN_CYPH_LEN + vpslldq xmm15, xmm15, 8 ; xmm15 = len(A)|| 0x0000000000000000 + vpxor xmm15, xmm15, xmm1 ; xmm15 = len(A)||len(C) + + vpxor xmm14, xmm15 + GHASH_MUL xmm14, xmm13, xmm0, xmm10, xmm11, xmm5, xmm6 + vpshufb xmm14, [rel SHUF_MASK] ; perform a 16Byte swap + + vpxor xmm9, xmm9, xmm14 + + +%%_return_T: + mov r10, %%AUTH_TAG ; r10 = authTag + mov r11, %%AUTH_TAG_LEN ; r11 = auth_tag_len + + cmp r11, 16 + je %%_T_16 + + cmp r11, 12 + je %%_T_12 + + cmp r11, 8 + je %%_T_8 + + simd_store_avx_15 r10, xmm9, r11, r12, rax + jmp %%_return_T_done +%%_T_8: + vmovq rax, xmm9 + mov [r10], rax + jmp %%_return_T_done +%%_T_12: + vmovq rax, xmm9 + mov [r10], rax + vpsrldq xmm9, xmm9, 8 + vmovd eax, xmm9 + mov [r10 + 8], eax + jmp %%_return_T_done +%%_T_16: + vmovdqu [r10], xmm9 + +%%_return_T_done: + +%ifdef SAFE_DATA + ;; Clear sensitive data from context structure + vpxor xmm0, xmm0 + vmovdqu [%%GDATA_CTX + AadHash], xmm0 + vmovdqu [%%GDATA_CTX + PBlockEncKey], xmm0 +%endif +%endmacro ; GCM_COMPLETE + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;void aes_gcm_precomp_128_vaes_avx512 / +; aes_gcm_precomp_192_vaes_avx512 / +; aes_gcm_precomp_256_vaes_avx512 +; (struct gcm_key_data *key_data) +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +MKGLOBAL(FN_NAME(precomp,_),function,) +FN_NAME(precomp,_): +;; Parameter is passed through register +%ifdef SAFE_PARAM + ;; Check key_data != NULL + cmp arg1, 0 + jz exit_precomp +%endif + + FUNC_SAVE + + vpxor xmm6, xmm6 + ENCRYPT_SINGLE_BLOCK arg1, xmm6 ; xmm6 = HashKey + + vpshufb xmm6, [rel SHUF_MASK] + ;;;;;;;;;;;;;;; PRECOMPUTATION of HashKey<<1 mod poly from the HashKey;;;;;;;;;;;;;;; + vmovdqa xmm2, xmm6 + vpsllq xmm6, xmm6, 1 + vpsrlq xmm2, xmm2, 63 + vmovdqa xmm1, xmm2 + vpslldq xmm2, xmm2, 8 + vpsrldq xmm1, xmm1, 8 + vpor xmm6, xmm6, xmm2 + ;reduction + vpshufd xmm2, xmm1, 00100100b + vpcmpeqd xmm2, [rel TWOONE] + vpand xmm2, xmm2, [rel POLY] + vpxor xmm6, xmm6, xmm2 ; xmm6 holds the HashKey<<1 mod poly + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + vmovdqu [arg1 + HashKey], xmm6 ; store HashKey<<1 mod poly + + + PRECOMPUTE arg1, xmm6, xmm0, xmm1, xmm2, xmm3, xmm4, xmm5 + + FUNC_RESTORE +exit_precomp: + + ret + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;void aes_gcm_init_128_vaes_avx512 / aes_gcm_init_192_vaes_avx512 / aes_gcm_init_256_vaes_avx512 +; (const struct gcm_key_data *key_data, +; struct gcm_context_data *context_data, +; u8 *iv, +; const u8 *aad, +; u64 aad_len); +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +MKGLOBAL(FN_NAME(init,_),function,) +FN_NAME(init,_): + FUNC_SAVE + +%ifdef SAFE_PARAM + ;; Check key_data != NULL + cmp arg1, 0 + jz exit_init + + ;; Check context_data != NULL + cmp arg2, 0 + jz exit_init + + ;; Check IV != NULL + cmp arg3, 0 + jz exit_init + + ;; Check if aad_len == 0 + cmp arg5, 0 + jz skip_aad_check_init + + ;; Check aad != NULL (aad_len != 0) + cmp arg4, 0 + jz exit_init + +skip_aad_check_init: +%endif + GCM_INIT arg1, arg2, arg3, arg4, arg5, r10, r11, r12, k1, xmm14, xmm2, \ + zmm1, zmm2, zmm3, zmm4, zmm5, zmm6, zmm7, zmm8, zmm9, zmm10 + +exit_init: + + FUNC_RESTORE + ret + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;void aes_gcm_enc_128_update_vaes_avx512 / aes_gcm_enc_192_update_vaes_avx512 / +; aes_gcm_enc_256_update_vaes_avx512 +; (const struct gcm_key_data *key_data, +; struct gcm_context_data *context_data, +; u8 *out, +; const u8 *in, +; u64 plaintext_len); +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +MKGLOBAL(FN_NAME(enc,_update_),function,) +FN_NAME(enc,_update_): + FUNC_SAVE + +%ifdef SAFE_PARAM + ;; Check key_data != NULL + cmp arg1, 0 + jz exit_update_enc + + ;; Check context_data != NULL + cmp arg2, 0 + jz exit_update_enc + + ;; Check if plaintext_len == 0 + cmp arg5, 0 + jz skip_in_out_check_update_enc + + ;; Check out != NULL (plaintext_len != 0) + cmp arg3, 0 + jz exit_update_enc + + ;; Check in != NULL (plaintext_len != 0) + cmp arg4, 0 + jz exit_update_enc + +skip_in_out_check_update_enc: +%endif + GCM_ENC_DEC arg1, arg2, arg3, arg4, arg5, ENC, multi_call + +exit_update_enc: + FUNC_RESTORE + ret + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;void aes_gcm_dec_128_update_vaes_avx512 / aes_gcm_dec_192_update_vaes_avx512 / +; aes_gcm_dec_256_update_vaes_avx512 +; (const struct gcm_key_data *key_data, +; struct gcm_context_data *context_data, +; u8 *out, +; const u8 *in, +; u64 plaintext_len); +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +MKGLOBAL(FN_NAME(dec,_update_),function,) +FN_NAME(dec,_update_): + FUNC_SAVE + +%ifdef SAFE_PARAM + ;; Check key_data != NULL + cmp arg1, 0 + jz exit_update_dec + + ;; Check context_data != NULL + cmp arg2, 0 + jz exit_update_dec + + ;; Check if plaintext_len == 0 + cmp arg5, 0 + jz skip_in_out_check_update_dec + + ;; Check out != NULL (plaintext_len != 0) + cmp arg3, 0 + jz exit_update_dec + + ;; Check in != NULL (plaintext_len != 0) + cmp arg4, 0 + jz exit_update_dec + +skip_in_out_check_update_dec: +%endif + + GCM_ENC_DEC arg1, arg2, arg3, arg4, arg5, DEC, multi_call + +exit_update_dec: + FUNC_RESTORE + ret + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;void aes_gcm_enc_128_finalize_vaes_avx512 / aes_gcm_enc_192_finalize_vaes_avx512 / +; aes_gcm_enc_256_finalize_vaes_avx512 +; (const struct gcm_key_data *key_data, +; struct gcm_context_data *context_data, +; u8 *auth_tag, +; u64 auth_tag_len); +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +MKGLOBAL(FN_NAME(enc,_finalize_),function,) +FN_NAME(enc,_finalize_): + +;; All parameters are passed through registers +%ifdef SAFE_PARAM + ;; Check key_data != NULL + cmp arg1, 0 + jz exit_enc_fin + + ;; Check context_data != NULL + cmp arg2, 0 + jz exit_enc_fin + + ;; Check auth_tag != NULL + cmp arg3, 0 + jz exit_enc_fin + + ;; Check auth_tag_len == 0 or > 16 + cmp arg4, 0 + jz exit_enc_fin + + cmp arg4, 16 + ja exit_enc_fin +%endif + + FUNC_SAVE + GCM_COMPLETE arg1, arg2, arg3, arg4, ENC, multi_call + + FUNC_RESTORE + +exit_enc_fin: + ret + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;void aes_gcm_dec_128_finalize_vaes_avx512 / aes_gcm_dec_192_finalize_vaes_avx512 +; aes_gcm_dec_256_finalize_vaes_avx512 +; (const struct gcm_key_data *key_data, +; struct gcm_context_data *context_data, +; u8 *auth_tag, +; u64 auth_tag_len); +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +MKGLOBAL(FN_NAME(dec,_finalize_),function,) +FN_NAME(dec,_finalize_): + +;; All parameters are passed through registers +%ifdef SAFE_PARAM + ;; Check key_data != NULL + cmp arg1, 0 + jz exit_dec_fin + + ;; Check context_data != NULL + cmp arg2, 0 + jz exit_dec_fin + + ;; Check auth_tag != NULL + cmp arg3, 0 + jz exit_dec_fin + + ;; Check auth_tag_len == 0 or > 16 + cmp arg4, 0 + jz exit_dec_fin + + cmp arg4, 16 + ja exit_dec_fin +%endif + + FUNC_SAVE + GCM_COMPLETE arg1, arg2, arg3, arg4, DEC, multi_call + + FUNC_RESTORE + +exit_dec_fin: + ret + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;void aes_gcm_enc_128_vaes_avx512 / aes_gcm_enc_192_vaes_avx512 / aes_gcm_enc_256_vaes_avx512 +; (const struct gcm_key_data *key_data, +; struct gcm_context_data *context_data, +; u8 *out, +; const u8 *in, +; u64 plaintext_len, +; u8 *iv, +; const u8 *aad, +; u64 aad_len, +; u8 *auth_tag, +; u64 auth_tag_len); +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +MKGLOBAL(FN_NAME(enc,_),function,) +FN_NAME(enc,_): + + FUNC_SAVE + +%ifdef SAFE_PARAM + ;; Check key_data != NULL + cmp arg1, 0 + jz exit_enc + + ;; Check context_data != NULL + cmp arg2, 0 + jz exit_enc + + ;; Check IV != NULL + cmp arg6, 0 + jz exit_enc + + ;; Check auth_tag != NULL + cmp arg9, 0 + jz exit_enc + + ;; Check auth_tag_len == 0 or > 16 + cmp arg10, 0 + jz exit_enc + + cmp arg10, 16 + ja exit_enc + + ;; Check if plaintext_len == 0 + cmp arg5, 0 + jz skip_in_out_check_enc + + ;; Check out != NULL (plaintext_len != 0) + cmp arg3, 0 + jz exit_enc + + ;; Check in != NULL (plaintext_len != 0) + cmp arg4, 0 + jz exit_enc + +skip_in_out_check_enc: + ;; Check if aad_len == 0 + cmp arg8, 0 + jz skip_aad_check_enc + + ;; Check aad != NULL (aad_len != 0) + cmp arg7, 0 + jz exit_enc + +skip_aad_check_enc: +%endif + GCM_INIT arg1, arg2, arg6, arg7, arg8, r10, r11, r12, k1, xmm14, xmm2, \ + zmm1, zmm2, zmm3, zmm4, zmm5, zmm6, zmm7, zmm8, zmm9, zmm10 + GCM_ENC_DEC arg1, arg2, arg3, arg4, arg5, ENC, single_call + GCM_COMPLETE arg1, arg2, arg9, arg10, ENC, single_call + +exit_enc: + FUNC_RESTORE + ret + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;void aes_gcm_dec_128_vaes_avx512 / aes_gcm_dec_192_vaes_avx512 / aes_gcm_dec_256_vaes_avx512 +; (const struct gcm_key_data *key_data, +; struct gcm_context_data *context_data, +; u8 *out, +; const u8 *in, +; u64 plaintext_len, +; u8 *iv, +; const u8 *aad, +; u64 aad_len, +; u8 *auth_tag, +; u64 auth_tag_len); +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +MKGLOBAL(FN_NAME(dec,_),function,) +FN_NAME(dec,_): + + FUNC_SAVE + +%ifdef SAFE_PARAM + ;; Check key_data != NULL + cmp arg1, 0 + jz exit_dec + + ;; Check context_data != NULL + cmp arg2, 0 + jz exit_dec + + ;; Check IV != NULL + cmp arg6, 0 + jz exit_dec + + ;; Check auth_tag != NULL + cmp arg9, 0 + jz exit_dec + + ;; Check auth_tag_len == 0 or > 16 + cmp arg10, 0 + jz exit_dec + + cmp arg10, 16 + ja exit_dec + + ;; Check if plaintext_len == 0 + cmp arg5, 0 + jz skip_in_out_check_dec + + ;; Check out != NULL (plaintext_len != 0) + cmp arg3, 0 + jz exit_dec + + ;; Check in != NULL (plaintext_len != 0) + cmp arg4, 0 + jz exit_dec + +skip_in_out_check_dec: + ;; Check if aad_len == 0 + cmp arg8, 0 + jz skip_aad_check_dec + + ;; Check aad != NULL (aad_len != 0) + cmp arg7, 0 + jz exit_dec + +skip_aad_check_dec: +%endif + GCM_INIT arg1, arg2, arg6, arg7, arg8, r10, r11, r12, k1, xmm14, xmm2, \ + zmm1, zmm2, zmm3, zmm4, zmm5, zmm6, zmm7, zmm8, zmm9, zmm10 + GCM_ENC_DEC arg1, arg2, arg3, arg4, arg5, DEC, single_call + GCM_COMPLETE arg1, arg2, arg9, arg10, DEC, single_call + +exit_dec: + FUNC_RESTORE + ret + +%ifdef LINUX +section .note.GNU-stack noalloc noexec nowrite progbits +%endif diff --git a/src/spdk/intel-ipsec-mb/avx512/mb_mgr_aes192_flush_avx512.asm b/src/spdk/intel-ipsec-mb/avx512/mb_mgr_aes192_flush_avx512.asm new file mode 100644 index 000000000..449229531 --- /dev/null +++ b/src/spdk/intel-ipsec-mb/avx512/mb_mgr_aes192_flush_avx512.asm @@ -0,0 +1,31 @@ +;; +;; Copyright (c) 2019, Intel Corporation +;; +;; Redistribution and use in source and binary forms, with or without +;; modification, are permitted provided that the following conditions are met: +;; +;; * Redistributions of source code must retain the above copyright notice, +;; this list of conditions and the following disclaimer. +;; * Redistributions in binary form must reproduce the above copyright +;; notice, this list of conditions and the following disclaimer in the +;; documentation and/or other materials provided with the distribution. +;; * Neither the name of Intel Corporation nor the names of its contributors +;; may be used to endorse or promote products derived from this software +;; without specific prior written permission. +;; +;; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +;; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +;; IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +;; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE +;; FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +;; DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +;; SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +;; CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +;; OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +;; + +%define AES_CBC_ENC_X16 aes_cbc_enc_192_vaes_avx512 +%define FLUSH_JOB_AES_ENC flush_job_aes192_enc_vaes_avx512 +%define NUM_KEYS 13 +%include "avx512/mb_mgr_aes_flush_avx512.asm" diff --git a/src/spdk/intel-ipsec-mb/avx512/mb_mgr_aes192_submit_avx512.asm b/src/spdk/intel-ipsec-mb/avx512/mb_mgr_aes192_submit_avx512.asm new file mode 100644 index 000000000..3bbb30158 --- /dev/null +++ b/src/spdk/intel-ipsec-mb/avx512/mb_mgr_aes192_submit_avx512.asm @@ -0,0 +1,31 @@ +;; +;; Copyright (c) 2019, Intel Corporation +;; +;; Redistribution and use in source and binary forms, with or without +;; modification, are permitted provided that the following conditions are met: +;; +;; * Redistributions of source code must retain the above copyright notice, +;; this list of conditions and the following disclaimer. +;; * Redistributions in binary form must reproduce the above copyright +;; notice, this list of conditions and the following disclaimer in the +;; documentation and/or other materials provided with the distribution. +;; * Neither the name of Intel Corporation nor the names of its contributors +;; may be used to endorse or promote products derived from this software +;; without specific prior written permission. +;; +;; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +;; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +;; IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +;; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE +;; FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +;; DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +;; SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +;; CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +;; OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +;; + +%define AES_CBC_ENC_X16 aes_cbc_enc_192_vaes_avx512 +%define SUBMIT_JOB_AES_ENC submit_job_aes192_enc_vaes_avx512 +%define NUM_KEYS 13 +%include "avx512/mb_mgr_aes_submit_avx512.asm" diff --git a/src/spdk/intel-ipsec-mb/avx512/mb_mgr_aes256_flush_avx512.asm b/src/spdk/intel-ipsec-mb/avx512/mb_mgr_aes256_flush_avx512.asm new file mode 100644 index 000000000..2ff448393 --- /dev/null +++ b/src/spdk/intel-ipsec-mb/avx512/mb_mgr_aes256_flush_avx512.asm @@ -0,0 +1,31 @@ +;; +;; Copyright (c) 2019, Intel Corporation +;; +;; Redistribution and use in source and binary forms, with or without +;; modification, are permitted provided that the following conditions are met: +;; +;; * Redistributions of source code must retain the above copyright notice, +;; this list of conditions and the following disclaimer. +;; * Redistributions in binary form must reproduce the above copyright +;; notice, this list of conditions and the following disclaimer in the +;; documentation and/or other materials provided with the distribution. +;; * Neither the name of Intel Corporation nor the names of its contributors +;; may be used to endorse or promote products derived from this software +;; without specific prior written permission. +;; +;; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +;; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +;; IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +;; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE +;; FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +;; DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +;; SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +;; CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +;; OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +;; + +%define AES_CBC_ENC_X16 aes_cbc_enc_256_vaes_avx512 +%define FLUSH_JOB_AES_ENC flush_job_aes256_enc_vaes_avx512 +%define NUM_KEYS 15 +%include "avx512/mb_mgr_aes_flush_avx512.asm" diff --git a/src/spdk/intel-ipsec-mb/avx512/mb_mgr_aes256_submit_avx512.asm b/src/spdk/intel-ipsec-mb/avx512/mb_mgr_aes256_submit_avx512.asm new file mode 100644 index 000000000..4db4629e2 --- /dev/null +++ b/src/spdk/intel-ipsec-mb/avx512/mb_mgr_aes256_submit_avx512.asm @@ -0,0 +1,31 @@ +;; +;; Copyright (c) 2019, Intel Corporation +;; +;; Redistribution and use in source and binary forms, with or without +;; modification, are permitted provided that the following conditions are met: +;; +;; * Redistributions of source code must retain the above copyright notice, +;; this list of conditions and the following disclaimer. +;; * Redistributions in binary form must reproduce the above copyright +;; notice, this list of conditions and the following disclaimer in the +;; documentation and/or other materials provided with the distribution. +;; * Neither the name of Intel Corporation nor the names of its contributors +;; may be used to endorse or promote products derived from this software +;; without specific prior written permission. +;; +;; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +;; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +;; IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +;; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE +;; FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +;; DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +;; SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +;; CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +;; OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +;; + +%define AES_CBC_ENC_X16 aes_cbc_enc_256_vaes_avx512 +%define SUBMIT_JOB_AES_ENC submit_job_aes256_enc_vaes_avx512 +%define NUM_KEYS 15 +%include "avx512/mb_mgr_aes_submit_avx512.asm" diff --git a/src/spdk/intel-ipsec-mb/avx512/mb_mgr_aes_flush_avx512.asm b/src/spdk/intel-ipsec-mb/avx512/mb_mgr_aes_flush_avx512.asm new file mode 100644 index 000000000..4a52ed1e6 --- /dev/null +++ b/src/spdk/intel-ipsec-mb/avx512/mb_mgr_aes_flush_avx512.asm @@ -0,0 +1,320 @@ +;; +;; Copyright (c) 2019, Intel Corporation +;; +;; Redistribution and use in source and binary forms, with or without +;; modification, are permitted provided that the following conditions are met: +;; +;; * Redistributions of source code must retain the above copyright notice, +;; this list of conditions and the following disclaimer. +;; * Redistributions in binary form must reproduce the above copyright +;; notice, this list of conditions and the following disclaimer in the +;; documentation and/or other materials provided with the distribution. +;; * Neither the name of Intel Corporation nor the names of its contributors +;; may be used to endorse or promote products derived from this software +;; without specific prior written permission. +;; +;; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +;; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +;; IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +;; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE +;; FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +;; DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +;; SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +;; CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +;; OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +;; + +%include "include/os.asm" +%include "job_aes_hmac.asm" +%include "mb_mgr_datastruct.asm" +%include "constants.asm" +%include "include/reg_sizes.asm" + +%ifndef AES_CBC_ENC_X16 +%define AES_CBC_ENC_X16 aes_cbc_enc_128_vaes_avx512 +%define FLUSH_JOB_AES_ENC flush_job_aes128_enc_vaes_avx512 +%define NUM_KEYS 11 +%endif + +; void AES_CBC_ENC_X16(AES_ARGS *args, UINT64 len_in_bytes); +extern AES_CBC_ENC_X16 + +section .text + +%define APPEND(a,b) a %+ b + +%ifdef LINUX +%define arg1 rdi +%define arg2 rsi +%else +%define arg1 rcx +%define arg2 rdx +%endif + +%define state arg1 +%define job arg2 +%define len2 arg2 + +%define job_rax rax + +%if 1 +%define unused_lanes rbx +%define tmp1 rbx + +%define good_lane rdx +%define iv rdx + +%define tmp2 rax + +; idx needs to be in rbp +%define tmp rbp +%define idx rbp + +%define tmp3 r8 +%define tmp4 r9 +%endif + +; copy IV into NULL lanes +%macro COPY_IV_TO_NULL_LANES 4 +%define %%IDX %1 ; [in] GP with good lane idx (scaled x16) +%define %%NULL_MASK %2 ; [clobbered] GP to store NULL lane mask +%define %%XTMP %3 ; [clobbered] temp XMM reg +%define %%MASK_REG %4 ; [in] mask register + + vmovdqa64 %%XTMP, [state + _aes_args_IV + %%IDX] + kmovw DWORD(%%NULL_MASK), %%MASK_REG +%assign i 0 +%rep 16 + bt %%NULL_MASK, i + jnc %%_skip_copy %+ i + vmovdqa64 [state + _aes_args_IV + (i*16)], %%XTMP +%%_skip_copy %+ i: +%assign i (i + 1) +%endrep + +%endmacro + +; clear IV into NULL lanes +%macro CLEAR_IV_IN_NULL_LANES 3 +%define %%NULL_MASK %1 ; [clobbered] GP to store NULL lane mask +%define %%XTMP %2 ; [clobbered] temp XMM reg +%define %%MASK_REG %3 ; [in] mask register + + vpxorq %%XTMP, %%XTMP + kmovw DWORD(%%NULL_MASK), %%MASK_REG +%assign i 0 +%rep 16 + bt %%NULL_MASK, i + jnc %%_skip_clear %+ i + vmovdqa64 [state + _aes_args_IV + (i*16)], %%XTMP +%%_skip_clear %+ i: +%assign i (i + 1) +%endrep + +%endmacro + +; copy round key's into NULL lanes +%macro COPY_KEYS_TO_NULL_LANES 5 +%define %%IDX %1 ; [in] GP with good lane idx (scaled x16) +%define %%NULL_MASK %2 ; [clobbered] GP to store NULL lane mask +%define %%KEY_TAB %3 ; [clobbered] GP to store key table pointer +%define %%XTMP %4 ; [clobbered] temp XMM reg +%define %%MASK_REG %5 ; [in] mask register + + lea %%KEY_TAB, [state + _aes_args_key_tab] + kmovw DWORD(%%NULL_MASK), %%MASK_REG +%assign j 0 ; outer loop to iterate through round keys +%rep 15 + vmovdqa64 %%XTMP, [%%KEY_TAB + j + %%IDX] +%assign k 0 ; inner loop to iterate through lanes +%rep 16 + bt %%NULL_MASK, k + jnc %%_skip_copy %+ j %+ _ %+ k + vmovdqa64 [%%KEY_TAB + j + (k*16)], %%XTMP +%%_skip_copy %+ j %+ _ %+ k: +%assign k (k + 1) +%endrep + +%assign j (j + 256) +%endrep + +%endmacro + +; clear round key's in NULL lanes +%macro CLEAR_KEYS_IN_NULL_LANES 3 +%define %%NULL_MASK %1 ; [clobbered] GP to store NULL lane mask +%define %%XTMP %2 ; [clobbered] temp XMM reg +%define %%MASK_REG %3 ; [in] mask register + + vpxorq %%XTMP, %%XTMP + kmovw DWORD(%%NULL_MASK), %%MASK_REG +%assign k 0 ; outer loop to iterate through lanes +%rep 16 + bt %%NULL_MASK, k + jnc %%_skip_clear %+ k +%assign j 0 ; inner loop to iterate through round keys +%rep NUM_KEYS + vmovdqa64 [state + _aesarg_key_tab + j + (k*16)], %%XTMP +%assign j (j + 256) +%endrep +%%_skip_clear %+ k: +%assign k (k + 1) +%endrep + +%endmacro + +; STACK_SPACE needs to be an odd multiple of 8 +; This routine and its callee clobbers all GPRs +struc STACK +_gpr_save: resq 8 +_rsp_save: resq 1 +endstruc + +; JOB* FLUSH_JOB_AES_ENC(MB_MGR_AES_OOO *state, JOB_AES_HMAC *job) +; arg 1 : state +; arg 2 : job +MKGLOBAL(FLUSH_JOB_AES_ENC,function,internal) +FLUSH_JOB_AES_ENC: + + mov rax, rsp + sub rsp, STACK_size + and rsp, -16 + + mov [rsp + _gpr_save + 8*0], rbx + mov [rsp + _gpr_save + 8*1], rbp + mov [rsp + _gpr_save + 8*2], r12 + mov [rsp + _gpr_save + 8*3], r13 + mov [rsp + _gpr_save + 8*4], r14 + mov [rsp + _gpr_save + 8*5], r15 +%ifndef LINUX + mov [rsp + _gpr_save + 8*6], rsi + mov [rsp + _gpr_save + 8*7], rdi +%endif + mov [rsp + _rsp_save], rax ; original SP + + ; check for empty + cmp qword [state + _aes_lanes_in_use], 0 + je return_null + + ; find a lane with a non-null job + vpxord zmm0, zmm0, zmm0 + vmovdqu64 zmm1, [state + _aes_job_in_lane + (0*PTR_SZ)] + vmovdqu64 zmm2, [state + _aes_job_in_lane + (8*PTR_SZ)] + vpcmpq k1, zmm1, zmm0, 4 ; NEQ + vpcmpq k2, zmm2, zmm0, 4 ; NEQ + kmovw DWORD(tmp), k1 + kmovw DWORD(tmp1), k2 + mov DWORD(tmp2), DWORD(tmp1) + shl DWORD(tmp2), 8 + or DWORD(tmp2), DWORD(tmp) ; mask of non-null jobs in tmp2 + not BYTE(tmp) + kmovw k4, DWORD(tmp) + not BYTE(tmp1) + kmovw k5, DWORD(tmp1) + mov DWORD(tmp), DWORD(tmp2) + not WORD(tmp) + kmovw k6, DWORD(tmp) ; mask of NULL jobs in k4, k5 and k6 + mov DWORD(tmp), DWORD(tmp2) + xor tmp2, tmp2 + bsf WORD(tmp2), WORD(tmp) ; index of the 1st set bit in tmp2 + + ;; copy good lane data into NULL lanes + mov tmp, [state + _aes_args_in + tmp2*8] + vpbroadcastq zmm1, tmp + vmovdqa64 [state + _aes_args_in + (0*PTR_SZ)]{k4}, zmm1 + vmovdqa64 [state + _aes_args_in + (8*PTR_SZ)]{k5}, zmm1 + ;; - out pointer + mov tmp, [state + _aes_args_out + tmp2*8] + vpbroadcastq zmm1, tmp + vmovdqa64 [state + _aes_args_out + (0*PTR_SZ)]{k4}, zmm1 + vmovdqa64 [state + _aes_args_out + (8*PTR_SZ)]{k5}, zmm1 + + ;; - set len to UINT16_MAX + mov WORD(tmp), 0xffff + vpbroadcastw ymm3, WORD(tmp) + vmovdqa64 ymm0, [state + _aes_lens] + vmovdqu16 ymm0{k6}, ymm3 + vmovdqa64 [state + _aes_lens], ymm0 + + ;; Find min length for lanes 0-7 + vphminposuw xmm2, xmm0 + + ;; scale up good lane idx before copying IV and keys + shl tmp2, 4 + ;; - copy IV to null lanes + COPY_IV_TO_NULL_LANES tmp2, tmp1, xmm4, k6 + + ; extract min length of lanes 0-7 + vpextrw DWORD(len2), xmm2, 0 ; min value + vpextrw DWORD(idx), xmm2, 1 ; min index + + ;; - copy round keys to null lanes + COPY_KEYS_TO_NULL_LANES tmp2, tmp1, tmp3, xmm4, k6 + + ;; Update lens and find min for lanes 8-15 + vextracti128 xmm1, ymm0, 1 + vphminposuw xmm2, xmm1 + vpextrw DWORD(tmp3), xmm2, 0 ; min value + cmp DWORD(len2), DWORD(tmp3) + jle use_min + vpextrw DWORD(idx), xmm2, 1 ; min index + add DWORD(idx), 8 ; but index +8 + mov len2, tmp3 ; min len +use_min: + vpbroadcastw ymm3, WORD(len2) + vpsubw ymm0, ymm0, ymm3 + vmovdqa [state + _aes_lens], ymm0 + + ; "state" and "args" are the same address, arg1 + ; len is arg2 + call AES_CBC_ENC_X16 + ; state and idx are intact + +len_is_0: + ; process completed job "idx" + mov job_rax, [state + _aes_job_in_lane + idx*8] + mov unused_lanes, [state + _aes_unused_lanes] + mov qword [state + _aes_job_in_lane + idx*8], 0 + or dword [job_rax + _status], STS_COMPLETED_AES + shl unused_lanes, 4 + or unused_lanes, idx + mov [state + _aes_unused_lanes], unused_lanes + sub qword [state + _aes_lanes_in_use], 1 + +%ifdef SAFE_DATA + ; Set bit of lane of returned job + xor DWORD(tmp3), DWORD(tmp3) + bts DWORD(tmp3), DWORD(idx) + kmovw k1, DWORD(tmp3) + korw k6, k1, k6 + + ;; Clear IV and expanded keys of returned job and "NULL lanes" + ;; (k6 contains the mask of the jobs) + CLEAR_IV_IN_NULL_LANES tmp1, xmm0, k6 + CLEAR_KEYS_IN_NULL_LANES tmp1, xmm0, k6 +%endif + +return: + + mov rbx, [rsp + _gpr_save + 8*0] + mov rbp, [rsp + _gpr_save + 8*1] + mov r12, [rsp + _gpr_save + 8*2] + mov r13, [rsp + _gpr_save + 8*3] + mov r14, [rsp + _gpr_save + 8*4] + mov r15, [rsp + _gpr_save + 8*5] +%ifndef LINUX + mov rsi, [rsp + _gpr_save + 8*6] + mov rdi, [rsp + _gpr_save + 8*7] +%endif + mov rsp, [rsp + _rsp_save] ; original SP + + ret + +return_null: + xor job_rax, job_rax + jmp return + +%ifdef LINUX +section .note.GNU-stack noalloc noexec nowrite progbits +%endif diff --git a/src/spdk/intel-ipsec-mb/avx512/mb_mgr_aes_submit_avx512.asm b/src/spdk/intel-ipsec-mb/avx512/mb_mgr_aes_submit_avx512.asm new file mode 100644 index 000000000..f79d15f68 --- /dev/null +++ b/src/spdk/intel-ipsec-mb/avx512/mb_mgr_aes_submit_avx512.asm @@ -0,0 +1,280 @@ +;; +;; Copyright (c) 2019, Intel Corporation +;; +;; Redistribution and use in source and binary forms, with or without +;; modification, are permitted provided that the following conditions are met: +;; +;; * Redistributions of source code must retain the above copyright notice, +;; this list of conditions and the following disclaimer. +;; * Redistributions in binary form must reproduce the above copyright +;; notice, this list of conditions and the following disclaimer in the +;; documentation and/or other materials provided with the distribution. +;; * Neither the name of Intel Corporation nor the names of its contributors +;; may be used to endorse or promote products derived from this software +;; without specific prior written permission. +;; +;; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +;; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +;; IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +;; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE +;; FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +;; DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +;; SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +;; CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +;; OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +;; + +%include "include/os.asm" +%include "job_aes_hmac.asm" +%include "mb_mgr_datastruct.asm" + +%include "include/reg_sizes.asm" +%include "include/const.inc" +%ifndef AES_CBC_ENC_X16 +%define AES_CBC_ENC_X16 aes_cbc_enc_128_vaes_avx512 +%define NUM_KEYS 11 +%define SUBMIT_JOB_AES_ENC submit_job_aes128_enc_vaes_avx512 +%endif + +; void AES_CBC_ENC_X16(AES_ARGS_x16 *args, UINT64 len_in_bytes); +extern AES_CBC_ENC_X16 + +section .text + +%ifdef LINUX +%define arg1 rdi +%define arg2 rsi +%else +%define arg1 rcx +%define arg2 rdx +%endif + +%define state arg1 +%define job arg2 +%define len2 arg2 + +%define job_rax rax + +%if 1 +; idx needs to be in rbp +%define len rbp +%define idx rbp +%define tmp r10 +%define tmp2 r11 +%define tmp3 r12 + +%define lane r8 + +%define iv r9 + +%define unused_lanes rbx +%endif + +; STACK_SPACE needs to be an odd multiple of 8 +; This routine and its callee clobbers all GPRs +struc STACK +_gpr_save: resq 8 +_rsp_save: resq 1 +endstruc + + +%macro INSERT_KEYS 6 +%define %%KP %1 ; [in] GP reg with pointer to expanded keys +%define %%LANE %2 ; [in] GP reg with lane number +%define %%NKEYS %3 ; [in] number of round keys (numerical value) +%define %%COL %4 ; [clobbered] GP reg +%define %%ZTMP %5 ; [clobbered] ZMM reg +%define %%IA0 %6 ; [clobbered] GP reg + + +%assign ROW (16*16) + + mov %%COL, %%LANE + shl %%COL, 4 + lea %%IA0, [state + _aes_args_key_tab] + add %%COL, %%IA0 + + vmovdqu64 %%ZTMP, [%%KP] + vextracti64x2 [%%COL + ROW*0], %%ZTMP, 0 + vextracti64x2 [%%COL + ROW*1], %%ZTMP, 1 + vextracti64x2 [%%COL + ROW*2], %%ZTMP, 2 + vextracti64x2 [%%COL + ROW*3], %%ZTMP, 3 + + vmovdqu64 %%ZTMP, [%%KP + 64] + vextracti64x2 [%%COL + ROW*4], %%ZTMP, 0 + vextracti64x2 [%%COL + ROW*5], %%ZTMP, 1 + vextracti64x2 [%%COL + ROW*6], %%ZTMP, 2 + vextracti64x2 [%%COL + ROW*7], %%ZTMP, 3 + +%if %%NKEYS > 11 ; 192 or 256 - copy 4 more keys + vmovdqu64 %%ZTMP, [%%KP + 128] + vextracti64x2 [%%COL + ROW*11], %%ZTMP, 3 +%else ; 128 - copy 3 more keys + mov %%IA0, 0x3f + kmovq k1, %%IA0 + vmovdqu64 %%ZTMP{k1}{z}, [%%KP + 128] +%endif + vextracti64x2 [%%COL + ROW*8], %%ZTMP, 0 + vextracti64x2 [%%COL + ROW*9], %%ZTMP, 1 + vextracti64x2 [%%COL + ROW*10], %%ZTMP, 2 + +%if %%NKEYS == 15 ; 256 - 3 more keys + mov %%IA0, 0x3f + kmovq k1, %%IA0 + vmovdqu64 %%ZTMP{k1}{z}, [%%KP + 192] + vextracti64x2 [%%COL + ROW*12], %%ZTMP, 0 + vextracti64x2 [%%COL + ROW*13], %%ZTMP, 1 + vextracti64x2 [%%COL + ROW*14], %%ZTMP, 2 +%elif %%NKEYS == 13 ; 192 - 1 more key + mov %%IA0, 0x3 + kmovq k1, %%IA0 + vmovdqu64 %%ZTMP{k1}{z}, [%%KP + 192] + vextracti64x2 [%%COL + ROW*12], %%ZTMP, 0 +%endif +%endmacro + +; JOB* SUBMIT_JOB_AES_ENC(MB_MGR_AES_OOO *state, JOB_AES_HMAC *job) +; arg 1 : state +; arg 2 : job +MKGLOBAL(SUBMIT_JOB_AES_ENC,function,internal) +SUBMIT_JOB_AES_ENC: + + mov rax, rsp + sub rsp, STACK_size + and rsp, -16 + + mov [rsp + _gpr_save + 8*0], rbx + mov [rsp + _gpr_save + 8*1], rbp + mov [rsp + _gpr_save + 8*2], r12 + mov [rsp + _gpr_save + 8*3], r13 + mov [rsp + _gpr_save + 8*4], r14 + mov [rsp + _gpr_save + 8*5], r15 +%ifndef LINUX + mov [rsp + _gpr_save + 8*6], rsi + mov [rsp + _gpr_save + 8*7], rdi +%endif + mov [rsp + _rsp_save], rax ; original SP + + mov unused_lanes, [state + _aes_unused_lanes] + mov lane, unused_lanes + and lane, 0xF + shr unused_lanes, 4 + mov len, [job + _msg_len_to_cipher_in_bytes] + and len, -16 ; DOCSIS may pass size unaligned to block size + mov iv, [job + _iv] + mov [state + _aes_unused_lanes], unused_lanes + add qword [state + _aes_lanes_in_use], 1 + + mov [state + _aes_job_in_lane + lane*8], job + + ;; Update lane len + vmovdqa64 ymm0, [state + _aes_lens] + mov tmp2, rcx ; save rcx + mov rcx, lane + mov tmp, 1 + shl tmp, cl + mov rcx, tmp2 ; restore rcx + kmovq k1, tmp + + vpbroadcastw ymm1, WORD(len) + vmovdqu16 ymm0{k1}, ymm1 + vmovdqa64 [state + _aes_lens], ymm0 + + ;; Find min length for lanes 0-7 + vphminposuw xmm2, xmm0 + + ;; Update input pointer + mov tmp, [job + _src] + add tmp, [job + _cipher_start_src_offset_in_bytes] + vmovdqu xmm1, [iv] + mov [state + _aes_args_in + lane*8], tmp + + ;; Insert expanded keys + mov tmp, [job + _aes_enc_key_expanded] + INSERT_KEYS tmp, lane, NUM_KEYS, tmp2, zmm4, tmp3 + + ;; Update output pointer + mov tmp, [job + _dst] + mov [state + _aes_args_out + lane*8], tmp + shl lane, 4 ; multiply by 16 + vmovdqa [state + _aes_args_IV + lane], xmm1 + + cmp qword [state + _aes_lanes_in_use], 16 + jne return_null + + ; Find min length for lanes 8-15 + vpextrw DWORD(len2), xmm2, 0 ; min value + vpextrw DWORD(idx), xmm2, 1 ; min index + vextracti128 xmm1, ymm0, 1 + vphminposuw xmm2, xmm1 + vpextrw DWORD(tmp), xmm2, 0 ; min value + cmp DWORD(len2), DWORD(tmp) + jle use_min + vpextrw DWORD(idx), xmm2, 1 ; min index + add DWORD(idx), 8 ; but index +8 + mov len2, tmp ; min len +use_min: + cmp len2, 0 + je len_is_0 + + vpbroadcastw ymm3, WORD(len2) + vpsubw ymm0, ymm0, ymm3 + vmovdqa [state + _aes_lens], ymm0 + + ; "state" and "args" are the same address, arg1 + ; len is arg2 + call AES_CBC_ENC_X16 + ; state and idx are intact + +len_is_0: + ; process completed job "idx" + mov job_rax, [state + _aes_job_in_lane + idx*8] + + mov unused_lanes, [state + _aes_unused_lanes] + mov qword [state + _aes_job_in_lane + idx*8], 0 + or dword [job_rax + _status], STS_COMPLETED_AES + shl unused_lanes, 4 + or unused_lanes, idx + + mov [state + _aes_unused_lanes], unused_lanes + sub qword [state + _aes_lanes_in_use], 1 + +%ifdef SAFE_DATA + ;; Clear IV + vpxorq xmm0, xmm0 + shl idx, 4 ; multiply by 16 + vmovdqa [state + _aes_args_IV + idx], xmm0 + + ;; Clear expanded keys +%assign round 0 +%rep NUM_KEYS + vmovdqa [state + _aesarg_key_tab + round * (16*16) + idx], xmm0 +%assign round (round + 1) +%endrep + +%endif + +return: + + mov rbx, [rsp + _gpr_save + 8*0] + mov rbp, [rsp + _gpr_save + 8*1] + mov r12, [rsp + _gpr_save + 8*2] + mov r13, [rsp + _gpr_save + 8*3] + mov r14, [rsp + _gpr_save + 8*4] + mov r15, [rsp + _gpr_save + 8*5] +%ifndef LINUX + mov rsi, [rsp + _gpr_save + 8*6] + mov rdi, [rsp + _gpr_save + 8*7] +%endif + mov rsp, [rsp + _rsp_save] ; original SP + + ret + +return_null: + xor job_rax, job_rax + jmp return + +%ifdef LINUX +section .note.GNU-stack noalloc noexec nowrite progbits +%endif diff --git a/src/spdk/intel-ipsec-mb/avx512/mb_mgr_avx512.c b/src/spdk/intel-ipsec-mb/avx512/mb_mgr_avx512.c new file mode 100644 index 000000000..bd1aaef63 --- /dev/null +++ b/src/spdk/intel-ipsec-mb/avx512/mb_mgr_avx512.c @@ -0,0 +1,1066 @@ +/******************************************************************************* + Copyright (c) 2012-2019, Intel Corporation + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of Intel Corporation nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE + FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*******************************************************************************/ + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +#define AVX512 +#define CLEAR_SCRATCH_SIMD_REGS clear_scratch_zmms + +#include "intel-ipsec-mb.h" +#include "include/kasumi_internal.h" +#include "include/zuc_internal.h" +#include "include/snow3g.h" + +#include "save_xmms.h" +#include "asm.h" +#include "des.h" +#include "gcm.h" +#include "cpu_feature.h" +#include "noaesni.h" + +JOB_AES_HMAC *submit_job_aes128_enc_avx(MB_MGR_AES_OOO *state, + JOB_AES_HMAC *job); +JOB_AES_HMAC *flush_job_aes128_enc_avx(MB_MGR_AES_OOO *state); + +JOB_AES_HMAC *submit_job_aes192_enc_avx(MB_MGR_AES_OOO *state, + JOB_AES_HMAC *job); +JOB_AES_HMAC *flush_job_aes192_enc_avx(MB_MGR_AES_OOO *state); + +JOB_AES_HMAC *submit_job_aes256_enc_avx(MB_MGR_AES_OOO *state, + JOB_AES_HMAC *job); +JOB_AES_HMAC *flush_job_aes256_enc_avx(MB_MGR_AES_OOO *state); + +JOB_AES_HMAC *submit_job_aes_xcbc_avx(MB_MGR_AES_XCBC_OOO *state, + JOB_AES_HMAC *job); +JOB_AES_HMAC *flush_job_aes_xcbc_avx(MB_MGR_AES_XCBC_OOO *state); + +JOB_AES_HMAC *submit_job_aes128_enc_vaes_avx512(MB_MGR_AES_OOO *state, + JOB_AES_HMAC *job); + +JOB_AES_HMAC *flush_job_aes128_enc_vaes_avx512(MB_MGR_AES_OOO *state); + +JOB_AES_HMAC *submit_job_aes192_enc_vaes_avx512(MB_MGR_AES_OOO *state, + JOB_AES_HMAC *job); + +JOB_AES_HMAC *flush_job_aes192_enc_vaes_avx512(MB_MGR_AES_OOO *state); + +JOB_AES_HMAC *submit_job_aes256_enc_vaes_avx512(MB_MGR_AES_OOO *state, + JOB_AES_HMAC *job); + +JOB_AES_HMAC *flush_job_aes256_enc_vaes_avx512(MB_MGR_AES_OOO *state); + +JOB_AES_HMAC *submit_job_des_cbc_enc_avx512(MB_MGR_DES_OOO *state, + JOB_AES_HMAC *job); +JOB_AES_HMAC *flush_job_des_cbc_enc_avx512(MB_MGR_DES_OOO *state); + +JOB_AES_HMAC *submit_job_des_cbc_dec_avx512(MB_MGR_DES_OOO *state, + JOB_AES_HMAC *job); +JOB_AES_HMAC *flush_job_des_cbc_dec_avx512(MB_MGR_DES_OOO *state); + +JOB_AES_HMAC *submit_job_3des_cbc_enc_avx512(MB_MGR_DES_OOO *state, + JOB_AES_HMAC *job); +JOB_AES_HMAC *flush_job_3des_cbc_enc_avx512(MB_MGR_DES_OOO *state); + +JOB_AES_HMAC *submit_job_3des_cbc_dec_avx512(MB_MGR_DES_OOO *state, + JOB_AES_HMAC *job); +JOB_AES_HMAC *flush_job_3des_cbc_dec_avx512(MB_MGR_DES_OOO *state); + +JOB_AES_HMAC *submit_job_docsis_des_enc_avx512(MB_MGR_DES_OOO *state, + JOB_AES_HMAC *job); +JOB_AES_HMAC *flush_job_docsis_des_enc_avx512(MB_MGR_DES_OOO *state); + +JOB_AES_HMAC *submit_job_docsis_des_dec_avx512(MB_MGR_DES_OOO *state, + JOB_AES_HMAC *job); +JOB_AES_HMAC *flush_job_docsis_des_dec_avx512(MB_MGR_DES_OOO *state); + +JOB_AES_HMAC *submit_job_aes_cntr_avx(JOB_AES_HMAC *job); + +JOB_AES_HMAC *submit_job_aes_cntr_bit_avx(JOB_AES_HMAC *job); + +#define SAVE_XMMS save_xmms_avx +#define RESTORE_XMMS restore_xmms_avx + +#define SUBMIT_JOB_AES128_ENC submit_job_aes128_enc_avx512 +#define SUBMIT_JOB_AES128_DEC submit_job_aes128_dec_avx512 +#define FLUSH_JOB_AES128_ENC flush_job_aes128_enc_avx512 + +#define SUBMIT_JOB_AES192_ENC submit_job_aes192_enc_avx512 +#define SUBMIT_JOB_AES192_DEC submit_job_aes192_dec_avx512 +#define FLUSH_JOB_AES192_ENC flush_job_aes192_enc_avx512 + +#define SUBMIT_JOB_AES256_ENC submit_job_aes256_enc_avx512 +#define SUBMIT_JOB_AES256_DEC submit_job_aes256_dec_avx512 +#define FLUSH_JOB_AES256_ENC flush_job_aes256_enc_avx512 + +#define SUBMIT_JOB_AES_ECB_128_ENC submit_job_aes_ecb_128_enc_avx +#define SUBMIT_JOB_AES_ECB_128_DEC submit_job_aes_ecb_128_dec_avx +#define SUBMIT_JOB_AES_ECB_192_ENC submit_job_aes_ecb_192_enc_avx +#define SUBMIT_JOB_AES_ECB_192_DEC submit_job_aes_ecb_192_dec_avx +#define SUBMIT_JOB_AES_ECB_256_ENC submit_job_aes_ecb_256_enc_avx +#define SUBMIT_JOB_AES_ECB_256_DEC submit_job_aes_ecb_256_dec_avx + +#define SUBMIT_JOB_AES_CNTR submit_job_aes_cntr_avx512 +#define SUBMIT_JOB_AES_CNTR_BIT submit_job_aes_cntr_bit_avx512 + +#define AES_CBC_DEC_128 aes_cbc_dec_128_avx512 +#define AES_CBC_DEC_192 aes_cbc_dec_192_avx512 +#define AES_CBC_DEC_256 aes_cbc_dec_256_avx512 + +#define AES_CNTR_128 aes_cntr_128_avx +#define AES_CNTR_192 aes_cntr_192_avx +#define AES_CNTR_256 aes_cntr_256_avx + +#define AES_CNTR_CCM_128 aes_cntr_ccm_128_avx + +#define AES_ECB_ENC_128 aes_ecb_enc_128_avx +#define AES_ECB_ENC_192 aes_ecb_enc_192_avx +#define AES_ECB_ENC_256 aes_ecb_enc_256_avx +#define AES_ECB_DEC_128 aes_ecb_dec_128_avx +#define AES_ECB_DEC_192 aes_ecb_dec_192_avx +#define AES_ECB_DEC_256 aes_ecb_dec_256_avx + +#define SUBMIT_JOB_PON_ENC submit_job_pon_enc_avx +#define SUBMIT_JOB_PON_DEC submit_job_pon_dec_avx +#define SUBMIT_JOB_PON_ENC_NO_CTR submit_job_pon_enc_no_ctr_avx +#define SUBMIT_JOB_PON_DEC_NO_CTR submit_job_pon_dec_no_ctr_avx + +#define SUBMIT_JOB_AES_XCBC submit_job_aes_xcbc_avx +#define FLUSH_JOB_AES_XCBC flush_job_aes_xcbc_avx + +#define SUBMIT_JOB_DES_CBC_ENC submit_job_des_cbc_enc_avx512 +#define FLUSH_JOB_DES_CBC_ENC flush_job_des_cbc_enc_avx512 + +#define SUBMIT_JOB_DES_CBC_DEC submit_job_des_cbc_dec_avx512 +#define FLUSH_JOB_DES_CBC_DEC flush_job_des_cbc_dec_avx512 + +#define SUBMIT_JOB_3DES_CBC_ENC submit_job_3des_cbc_enc_avx512 +#define FLUSH_JOB_3DES_CBC_ENC flush_job_3des_cbc_enc_avx512 + +#define SUBMIT_JOB_3DES_CBC_DEC submit_job_3des_cbc_dec_avx512 +#define FLUSH_JOB_3DES_CBC_DEC flush_job_3des_cbc_dec_avx512 + +#define SUBMIT_JOB_DOCSIS_DES_ENC submit_job_docsis_des_enc_avx512 +#define FLUSH_JOB_DOCSIS_DES_ENC flush_job_docsis_des_enc_avx512 + +#define SUBMIT_JOB_DOCSIS_DES_DEC submit_job_docsis_des_dec_avx512 +#define FLUSH_JOB_DOCSIS_DES_DEC flush_job_docsis_des_dec_avx512 + +#define SUBMIT_JOB_AES_ENC SUBMIT_JOB_AES_ENC_AVX512 +#define FLUSH_JOB_AES_ENC FLUSH_JOB_AES_ENC_AVX512 +#define SUBMIT_JOB_AES_DEC SUBMIT_JOB_AES_DEC_AVX512 + +JOB_AES_HMAC *submit_job_hmac_avx512(MB_MGR_HMAC_SHA_1_OOO *state, + JOB_AES_HMAC *job); +JOB_AES_HMAC *flush_job_hmac_avx512(MB_MGR_HMAC_SHA_1_OOO *state); + +JOB_AES_HMAC *submit_job_hmac_sha_224_avx512(MB_MGR_HMAC_SHA_256_OOO *state, + JOB_AES_HMAC *job); +JOB_AES_HMAC *flush_job_hmac_sha_224_avx512(MB_MGR_HMAC_SHA_256_OOO *state); + +JOB_AES_HMAC *submit_job_hmac_sha_256_avx512(MB_MGR_HMAC_SHA_256_OOO *state, + JOB_AES_HMAC *job); +JOB_AES_HMAC *flush_job_hmac_sha_256_avx512(MB_MGR_HMAC_SHA_256_OOO *state); + +JOB_AES_HMAC *submit_job_hmac_sha_384_avx512(MB_MGR_HMAC_SHA_512_OOO *state, + JOB_AES_HMAC *job); +JOB_AES_HMAC *flush_job_hmac_sha_384_avx512(MB_MGR_HMAC_SHA_512_OOO *state); + +JOB_AES_HMAC *submit_job_hmac_sha_512_avx512(MB_MGR_HMAC_SHA_512_OOO *state, + JOB_AES_HMAC *job); +JOB_AES_HMAC *flush_job_hmac_sha_512_avx512(MB_MGR_HMAC_SHA_512_OOO *state); + +JOB_AES_HMAC *submit_job_hmac_md5_avx2(MB_MGR_HMAC_MD5_OOO *state, + JOB_AES_HMAC *job); +JOB_AES_HMAC *flush_job_hmac_md5_avx2(MB_MGR_HMAC_MD5_OOO *state); + +JOB_AES_HMAC *submit_job_aes_cmac_auth_avx(MB_MGR_CMAC_OOO *state, + JOB_AES_HMAC *job); + +JOB_AES_HMAC *flush_job_aes_cmac_auth_avx(MB_MGR_CMAC_OOO *state); + +JOB_AES_HMAC *submit_job_aes_ccm_auth_avx(MB_MGR_CCM_OOO *state, + JOB_AES_HMAC *job); + +JOB_AES_HMAC *flush_job_aes_ccm_auth_avx(MB_MGR_CCM_OOO *state); + +#define SUBMIT_JOB_HMAC submit_job_hmac_avx512 +#define FLUSH_JOB_HMAC flush_job_hmac_avx512 +#define SUBMIT_JOB_HMAC_SHA_224 submit_job_hmac_sha_224_avx512 +#define FLUSH_JOB_HMAC_SHA_224 flush_job_hmac_sha_224_avx512 +#define SUBMIT_JOB_HMAC_SHA_256 submit_job_hmac_sha_256_avx512 +#define FLUSH_JOB_HMAC_SHA_256 flush_job_hmac_sha_256_avx512 +#define SUBMIT_JOB_HMAC_SHA_384 submit_job_hmac_sha_384_avx512 +#define FLUSH_JOB_HMAC_SHA_384 flush_job_hmac_sha_384_avx512 +#define SUBMIT_JOB_HMAC_SHA_512 submit_job_hmac_sha_512_avx512 +#define FLUSH_JOB_HMAC_SHA_512 flush_job_hmac_sha_512_avx512 +#define SUBMIT_JOB_HMAC_MD5 submit_job_hmac_md5_avx2 +#define FLUSH_JOB_HMAC_MD5 flush_job_hmac_md5_avx2 + +#ifndef NO_GCM +#define AES_GCM_DEC_128 aes_gcm_dec_128_avx512 +#define AES_GCM_ENC_128 aes_gcm_enc_128_avx512 +#define AES_GCM_DEC_192 aes_gcm_dec_192_avx512 +#define AES_GCM_ENC_192 aes_gcm_enc_192_avx512 +#define AES_GCM_DEC_256 aes_gcm_dec_256_avx512 +#define AES_GCM_ENC_256 aes_gcm_enc_256_avx512 + +#define AES_GCM_DEC_128_VAES aes_gcm_dec_128_vaes_avx512 +#define AES_GCM_ENC_128_VAES aes_gcm_enc_128_vaes_avx512 +#define AES_GCM_DEC_192_VAES aes_gcm_dec_192_vaes_avx512 +#define AES_GCM_ENC_192_VAES aes_gcm_enc_192_vaes_avx512 +#define AES_GCM_DEC_256_VAES aes_gcm_dec_256_vaes_avx512 +#define AES_GCM_ENC_256_VAES aes_gcm_enc_256_vaes_avx512 + +#define SUBMIT_JOB_AES_GCM_DEC submit_job_aes_gcm_dec_avx512 +#define FLUSH_JOB_AES_GCM_DEC flush_job_aes_gcm_avx512 +#define SUBMIT_JOB_AES_GCM_ENC submit_job_aes_gcm_enc_avx512 +#define FLUSH_JOB_AES_GCM_ENC flush_job_aes_gcm_avx512 +#endif /* NO_GCM */ + +/* ====================================================================== */ + +#define SUBMIT_JOB submit_job_avx512 +#define FLUSH_JOB flush_job_avx512 +#define QUEUE_SIZE queue_size_avx512 +#define SUBMIT_JOB_NOCHECK submit_job_nocheck_avx512 +#define GET_NEXT_JOB get_next_job_avx512 +#define GET_COMPLETED_JOB get_completed_job_avx512 + +/* ====================================================================== */ + +#define SUBMIT_JOB_HASH SUBMIT_JOB_HASH_AVX512 +#define FLUSH_JOB_HASH FLUSH_JOB_HASH_AVX512 + +/* ====================================================================== */ + +#define AES_CFB_128_ONE aes_cfb_128_one_avx512 + +void aes128_cbc_mac_x8(AES_ARGS *args, uint64_t len); + +#define AES128_CBC_MAC aes128_cbc_mac_x8 + +#define FLUSH_JOB_AES_CCM_AUTH flush_job_aes_ccm_auth_avx +#define SUBMIT_JOB_AES_CCM_AUTH submit_job_aes_ccm_auth_avx + +#define FLUSH_JOB_AES_CMAC_AUTH flush_job_aes_cmac_auth_avx +#define SUBMIT_JOB_AES_CMAC_AUTH submit_job_aes_cmac_auth_avx + +/* ====================================================================== */ + +/* + * GCM submit / flush API for AVX512 arch + */ +#ifndef NO_GCM +static JOB_AES_HMAC * +plain_submit_gcm_dec_avx512(MB_MGR *state, JOB_AES_HMAC *job) +{ + DECLARE_ALIGNED(struct gcm_context_data ctx, 16); + (void) state; + + if (16 == job->aes_key_len_in_bytes) + AES_GCM_DEC_128(job->aes_dec_key_expanded, &ctx, job->dst, + job->src + + job->cipher_start_src_offset_in_bytes, + job->msg_len_to_cipher_in_bytes, + job->iv, + job->u.GCM.aad, job->u.GCM.aad_len_in_bytes, + job->auth_tag_output, + job->auth_tag_output_len_in_bytes); + else if (24 == job->aes_key_len_in_bytes) + AES_GCM_DEC_192(job->aes_dec_key_expanded, &ctx, job->dst, + job->src + + job->cipher_start_src_offset_in_bytes, + job->msg_len_to_cipher_in_bytes, + job->iv, + job->u.GCM.aad, job->u.GCM.aad_len_in_bytes, + job->auth_tag_output, + job->auth_tag_output_len_in_bytes); + else /* assume 32 bytes */ + AES_GCM_DEC_256(job->aes_dec_key_expanded, &ctx, job->dst, + job->src + + job->cipher_start_src_offset_in_bytes, + job->msg_len_to_cipher_in_bytes, + job->iv, + job->u.GCM.aad, job->u.GCM.aad_len_in_bytes, + job->auth_tag_output, + job->auth_tag_output_len_in_bytes); + + job->status = STS_COMPLETED; + return job; +} + +static JOB_AES_HMAC * +plain_submit_gcm_enc_avx512(MB_MGR *state, JOB_AES_HMAC *job) +{ + DECLARE_ALIGNED(struct gcm_context_data ctx, 16); + (void) state; + + if (16 == job->aes_key_len_in_bytes) + AES_GCM_ENC_128(job->aes_enc_key_expanded, &ctx, job->dst, + job->src + + job->cipher_start_src_offset_in_bytes, + job->msg_len_to_cipher_in_bytes, job->iv, + job->u.GCM.aad, job->u.GCM.aad_len_in_bytes, + job->auth_tag_output, + job->auth_tag_output_len_in_bytes); + else if (24 == job->aes_key_len_in_bytes) + AES_GCM_ENC_192(job->aes_enc_key_expanded, &ctx, job->dst, + job->src + + job->cipher_start_src_offset_in_bytes, + job->msg_len_to_cipher_in_bytes, job->iv, + job->u.GCM.aad, job->u.GCM.aad_len_in_bytes, + job->auth_tag_output, + job->auth_tag_output_len_in_bytes); + else /* assume 32 bytes */ + AES_GCM_ENC_256(job->aes_enc_key_expanded, &ctx, job->dst, + job->src + + job->cipher_start_src_offset_in_bytes, + job->msg_len_to_cipher_in_bytes, job->iv, + job->u.GCM.aad, job->u.GCM.aad_len_in_bytes, + job->auth_tag_output, + job->auth_tag_output_len_in_bytes); + + job->status = STS_COMPLETED; + return job; +} + +static JOB_AES_HMAC * +vaes_submit_gcm_dec_avx512(MB_MGR *state, JOB_AES_HMAC *job) +{ + DECLARE_ALIGNED(struct gcm_context_data ctx, 16); + (void) state; + + if (16 == job->aes_key_len_in_bytes) + AES_GCM_DEC_128_VAES(job->aes_dec_key_expanded, &ctx, job->dst, + job->src + + job->cipher_start_src_offset_in_bytes, + job->msg_len_to_cipher_in_bytes, + job->iv, + job->u.GCM.aad, + job->u.GCM.aad_len_in_bytes, + job->auth_tag_output, + job->auth_tag_output_len_in_bytes); + else if (24 == job->aes_key_len_in_bytes) + AES_GCM_DEC_192_VAES(job->aes_dec_key_expanded, &ctx, job->dst, + job->src + + job->cipher_start_src_offset_in_bytes, + job->msg_len_to_cipher_in_bytes, + job->iv, + job->u.GCM.aad, + job->u.GCM.aad_len_in_bytes, + job->auth_tag_output, + job->auth_tag_output_len_in_bytes); + else /* assume 32 bytes */ + AES_GCM_DEC_256_VAES(job->aes_dec_key_expanded, &ctx, job->dst, + job->src + + job->cipher_start_src_offset_in_bytes, + job->msg_len_to_cipher_in_bytes, + job->iv, + job->u.GCM.aad, + job->u.GCM.aad_len_in_bytes, + job->auth_tag_output, + job->auth_tag_output_len_in_bytes); + + job->status = STS_COMPLETED; + return job; +} + +static JOB_AES_HMAC * +vaes_submit_gcm_enc_avx512(MB_MGR *state, JOB_AES_HMAC *job) +{ + DECLARE_ALIGNED(struct gcm_context_data ctx, 16); + (void) state; + + if (16 == job->aes_key_len_in_bytes) + AES_GCM_ENC_128_VAES(job->aes_enc_key_expanded, &ctx, job->dst, + job->src + + job->cipher_start_src_offset_in_bytes, + job->msg_len_to_cipher_in_bytes, job->iv, + job->u.GCM.aad, + job->u.GCM.aad_len_in_bytes, + job->auth_tag_output, + job->auth_tag_output_len_in_bytes); + else if (24 == job->aes_key_len_in_bytes) + AES_GCM_ENC_192_VAES(job->aes_enc_key_expanded, &ctx, job->dst, + job->src + + job->cipher_start_src_offset_in_bytes, + job->msg_len_to_cipher_in_bytes, job->iv, + job->u.GCM.aad, + job->u.GCM.aad_len_in_bytes, + job->auth_tag_output, + job->auth_tag_output_len_in_bytes); + else /* assume 32 bytes */ + AES_GCM_ENC_256_VAES(job->aes_enc_key_expanded, &ctx, job->dst, + job->src + + job->cipher_start_src_offset_in_bytes, + job->msg_len_to_cipher_in_bytes, job->iv, + job->u.GCM.aad, + job->u.GCM.aad_len_in_bytes, + job->auth_tag_output, + job->auth_tag_output_len_in_bytes); + + job->status = STS_COMPLETED; + return job; +} + +static JOB_AES_HMAC * +flush_job_aes_gcm_avx512(MB_MGR *state, JOB_AES_HMAC *job) +{ + (void) state; + (void) job; + return NULL; +} + +static JOB_AES_HMAC *(*submit_job_aes_gcm_enc_avx512) + (MB_MGR *state, JOB_AES_HMAC *job) = plain_submit_gcm_enc_avx512; + +static JOB_AES_HMAC *(*submit_job_aes_gcm_dec_avx512) + (MB_MGR *state, JOB_AES_HMAC *job) = plain_submit_gcm_dec_avx512; + +#endif /* NO_GCM */ + +static JOB_AES_HMAC *(*submit_job_aes_cntr_avx512) + (JOB_AES_HMAC *job) = submit_job_aes_cntr_avx; +static JOB_AES_HMAC *(*submit_job_aes_cntr_bit_avx512) + (JOB_AES_HMAC *job) = submit_job_aes_cntr_bit_avx; + +static JOB_AES_HMAC * +vaes_submit_cntr_avx512(JOB_AES_HMAC *job) +{ + if (16 == job->aes_key_len_in_bytes) + aes_cntr_128_submit_vaes_avx512(job); + else if (24 == job->aes_key_len_in_bytes) + aes_cntr_192_submit_vaes_avx512(job); + else /* assume 32 bytes */ + aes_cntr_256_submit_vaes_avx512(job); + + job->status |= STS_COMPLETED_AES; + return job; +} + +static JOB_AES_HMAC * +vaes_submit_cntr_bit_avx512(JOB_AES_HMAC *job) +{ + if (16 == job->aes_key_len_in_bytes) + aes_cntr_bit_128_submit_vaes_avx512(job); + else if (24 == job->aes_key_len_in_bytes) + aes_cntr_bit_192_submit_vaes_avx512(job); + else /* assume 32 bytes */ + aes_cntr_bit_256_submit_vaes_avx512(job); + + job->status |= STS_COMPLETED_AES; + return job; +} + +/* ====================================================================== */ + +static JOB_AES_HMAC * +(*submit_job_aes128_enc_avx512) + (MB_MGR_AES_OOO *state, JOB_AES_HMAC *job) = submit_job_aes128_enc_avx; + +static JOB_AES_HMAC * +(*submit_job_aes192_enc_avx512) + (MB_MGR_AES_OOO *state, JOB_AES_HMAC *job) = submit_job_aes192_enc_avx; + +static JOB_AES_HMAC * +(*submit_job_aes256_enc_avx512) + (MB_MGR_AES_OOO *state, JOB_AES_HMAC *job) = submit_job_aes256_enc_avx; + +static JOB_AES_HMAC * +(*flush_job_aes128_enc_avx512) + (MB_MGR_AES_OOO *state) = flush_job_aes128_enc_avx; + +static JOB_AES_HMAC * +(*flush_job_aes192_enc_avx512) + (MB_MGR_AES_OOO *state) = flush_job_aes192_enc_avx; + +static JOB_AES_HMAC * +(*flush_job_aes256_enc_avx512) + (MB_MGR_AES_OOO *state) = flush_job_aes256_enc_avx; + +static void +(*aes_cbc_dec_128_avx512) (const void *in, const uint8_t *IV, + const void *keys, void *out, + uint64_t len_bytes) = aes_cbc_dec_128_avx; +static void +(*aes_cbc_dec_192_avx512) (const void *in, const uint8_t *IV, + const void *keys, void *out, + uint64_t len_bytes) = aes_cbc_dec_192_avx; +static void +(*aes_cbc_dec_256_avx512) (const void *in, const uint8_t *IV, + const void *keys, void *out, + uint64_t len_bytes) = aes_cbc_dec_256_avx; + +void +init_mb_mgr_avx512(MB_MGR *state) +{ + unsigned int j, vaes_support = 0; + uint8_t *p; + size_t size; + + state->features = cpu_feature_adjust(state->flags, + cpu_feature_detect()); + + if (!(state->features & IMB_FEATURE_AESNI)) { + init_mb_mgr_sse_no_aesni(state); + return; + } + if ((state->features & IMB_FEATURE_VAES) == IMB_FEATURE_VAES) { + vaes_support = 1; + aes_cbc_dec_128_avx512 = aes_cbc_dec_128_vaes_avx512; + aes_cbc_dec_192_avx512 = aes_cbc_dec_192_vaes_avx512; + aes_cbc_dec_256_avx512 = aes_cbc_dec_256_vaes_avx512; + submit_job_aes128_enc_avx512 = + submit_job_aes128_enc_vaes_avx512; + flush_job_aes128_enc_avx512 = + flush_job_aes128_enc_vaes_avx512; + submit_job_aes192_enc_avx512 = + submit_job_aes192_enc_vaes_avx512; + flush_job_aes192_enc_avx512 = + flush_job_aes192_enc_vaes_avx512; + submit_job_aes256_enc_avx512 = + submit_job_aes256_enc_vaes_avx512; + flush_job_aes256_enc_avx512 = + flush_job_aes256_enc_vaes_avx512; + } + + /* Init AES out-of-order fields */ + if (vaes_support) { + /* init 16 lanes */ + memset(state->aes128_ooo.lens, 0, + sizeof(state->aes128_ooo.lens)); + memset(state->aes128_ooo.job_in_lane, 0, + sizeof(state->aes128_ooo.job_in_lane)); + state->aes128_ooo.unused_lanes = 0xFEDCBA9876543210; + state->aes128_ooo.num_lanes_inuse = 0; + + memset(state->aes192_ooo.lens, 0, + sizeof(state->aes192_ooo.lens)); + memset(state->aes192_ooo.job_in_lane, 0, + sizeof(state->aes192_ooo.job_in_lane)); + state->aes192_ooo.unused_lanes = 0xFEDCBA9876543210; + state->aes192_ooo.num_lanes_inuse = 0; + + memset(state->aes256_ooo.lens, 0, + sizeof(state->aes256_ooo.lens)); + memset(state->aes256_ooo.job_in_lane, 0, + sizeof(state->aes256_ooo.job_in_lane)); + state->aes256_ooo.unused_lanes = 0xFEDCBA9876543210; + state->aes256_ooo.num_lanes_inuse = 0; + } else { + /* init 8 lanes */ + memset(state->aes128_ooo.lens, 0xFF, + sizeof(state->aes128_ooo.lens)); + memset(&state->aes128_ooo.lens[0], 0, + sizeof(state->aes128_ooo.lens[0]) * 8); + memset(state->aes128_ooo.job_in_lane, 0, + sizeof(state->aes128_ooo.job_in_lane)); + state->aes128_ooo.unused_lanes = 0xF76543210; + state->aes128_ooo.num_lanes_inuse = 0; + + memset(state->aes192_ooo.lens, 0xFF, + sizeof(state->aes192_ooo.lens)); + memset(&state->aes192_ooo.lens[0], 0, + sizeof(state->aes192_ooo.lens[0]) * 8); + memset(state->aes192_ooo.job_in_lane, 0, + sizeof(state->aes192_ooo.job_in_lane)); + state->aes192_ooo.unused_lanes = 0xF76543210; + state->aes192_ooo.num_lanes_inuse = 0; + + memset(&state->aes256_ooo.lens, 0xFF, + sizeof(state->aes256_ooo.lens)); + memset(&state->aes256_ooo.lens[0], 0, + sizeof(state->aes256_ooo.lens[0]) * 8); + memset(state->aes256_ooo.job_in_lane, 0, + sizeof(state->aes256_ooo.job_in_lane)); + state->aes256_ooo.unused_lanes = 0xF76543210; + state->aes256_ooo.num_lanes_inuse = 0; + } + + + /* DOCSIS SEC BPI (AES CBC + AES CFB for partial block) + * uses same settings as AES128 CBC. + */ + if (vaes_support) { + /* init 16 lanes */ + memset(state->docsis_sec_ooo.lens, 0, + sizeof(state->docsis_sec_ooo.lens)); + memset(state->docsis_sec_ooo.job_in_lane, 0, + sizeof(state->docsis_sec_ooo.job_in_lane)); + state->docsis_sec_ooo.unused_lanes = 0xFEDCBA9876543210; + state->docsis_sec_ooo.num_lanes_inuse = 0; + } else { + /* init 8 lanes */ + memset(state->docsis_sec_ooo.lens, 0xFF, + sizeof(state->docsis_sec_ooo.lens)); + memset(&state->docsis_sec_ooo.lens[0], 0, + sizeof(state->docsis_sec_ooo.lens[0]) * 8); + memset(state->docsis_sec_ooo.job_in_lane, 0, + sizeof(state->docsis_sec_ooo.job_in_lane)); + state->docsis_sec_ooo.unused_lanes = 0xF76543210; + state->docsis_sec_ooo.num_lanes_inuse = 0; + } + + + /* DES, 3DES and DOCSIS DES (DES CBC + DES CFB for partial block) */ + /* - separate DES OOO for encryption */ + for (j = 0; j < AVX512_NUM_DES_LANES; j++) { + state->des_enc_ooo.lens[j] = 0; + state->des_enc_ooo.job_in_lane[j] = NULL; + } + state->des_enc_ooo.unused_lanes = 0xFEDCBA9876543210; + state->des_enc_ooo.num_lanes_inuse = 0; + memset(&state->des_enc_ooo.args, 0, sizeof(state->des_enc_ooo.args)); + + /* - separate DES OOO for decryption */ + for (j = 0; j < AVX512_NUM_DES_LANES; j++) { + state->des_dec_ooo.lens[j] = 0; + state->des_dec_ooo.job_in_lane[j] = NULL; + } + state->des_dec_ooo.unused_lanes = 0xFEDCBA9876543210; + state->des_dec_ooo.num_lanes_inuse = 0; + memset(&state->des_dec_ooo.args, 0, sizeof(state->des_dec_ooo.args)); + + /* - separate 3DES OOO for encryption */ + for (j = 0; j < AVX512_NUM_DES_LANES; j++) { + state->des3_enc_ooo.lens[j] = 0; + state->des3_enc_ooo.job_in_lane[j] = NULL; + } + state->des3_enc_ooo.unused_lanes = 0xFEDCBA9876543210; + state->des3_enc_ooo.num_lanes_inuse = 0; + memset(&state->des3_enc_ooo.args, 0, sizeof(state->des3_enc_ooo.args)); + + /* - separate 3DES OOO for decryption */ + for (j = 0; j < AVX512_NUM_DES_LANES; j++) { + state->des3_dec_ooo.lens[j] = 0; + state->des3_dec_ooo.job_in_lane[j] = NULL; + } + state->des3_dec_ooo.unused_lanes = 0xFEDCBA9876543210; + state->des3_dec_ooo.num_lanes_inuse = 0; + memset(&state->des3_dec_ooo.args, 0, sizeof(state->des3_dec_ooo.args)); + + /* - separate DOCSIS DES OOO for encryption */ + for (j = 0; j < AVX512_NUM_DES_LANES; j++) { + state->docsis_des_enc_ooo.lens[j] = 0; + state->docsis_des_enc_ooo.job_in_lane[j] = NULL; + } + state->docsis_des_enc_ooo.unused_lanes = 0xFEDCBA9876543210; + state->docsis_des_enc_ooo.num_lanes_inuse = 0; + memset(&state->docsis_des_enc_ooo.args, 0, + sizeof(state->docsis_des_enc_ooo.args)); + + /* - separate DES OOO for decryption */ + for (j = 0; j < AVX512_NUM_DES_LANES; j++) { + state->docsis_des_dec_ooo.lens[j] = 0; + state->docsis_des_dec_ooo.job_in_lane[j] = NULL; + } + state->docsis_des_dec_ooo.unused_lanes = 0xFEDCBA9876543210; + state->docsis_des_dec_ooo.num_lanes_inuse = 0; + memset(&state->docsis_des_dec_ooo.args, 0, + sizeof(state->docsis_des_dec_ooo.args)); + + /* Init HMAC/SHA1 out-of-order fields */ + state->hmac_sha_1_ooo.lens[0] = 0; + state->hmac_sha_1_ooo.lens[1] = 0; + state->hmac_sha_1_ooo.lens[2] = 0; + state->hmac_sha_1_ooo.lens[3] = 0; + state->hmac_sha_1_ooo.lens[4] = 0; + state->hmac_sha_1_ooo.lens[5] = 0; + state->hmac_sha_1_ooo.lens[6] = 0; + state->hmac_sha_1_ooo.lens[7] = 0; + state->hmac_sha_1_ooo.lens[8] = 0; + state->hmac_sha_1_ooo.lens[9] = 0; + state->hmac_sha_1_ooo.lens[10] = 0; + state->hmac_sha_1_ooo.lens[11] = 0; + state->hmac_sha_1_ooo.lens[12] = 0; + state->hmac_sha_1_ooo.lens[13] = 0; + state->hmac_sha_1_ooo.lens[14] = 0; + state->hmac_sha_1_ooo.lens[15] = 0; + state->hmac_sha_1_ooo.unused_lanes = 0xFEDCBA9876543210; + state->hmac_sha_1_ooo.num_lanes_inuse = 0; + for (j = 0; j < AVX512_NUM_SHA1_LANES; j++) { + state->hmac_sha_1_ooo.ldata[j].job_in_lane = NULL; + state->hmac_sha_1_ooo.ldata[j].extra_block[64] = 0x80; + memset(state->hmac_sha_1_ooo.ldata[j].extra_block + 65, + 0x00, + 64 + 7); + p = state->hmac_sha_1_ooo.ldata[j].outer_block; + memset(p + 5*4 + 1, + 0x00, + 64 - 5*4 - 1 - 2); + p[5 * 4] = 0x80; + p[64 - 2] = 0x02; + p[64 - 1] = 0xA0; + } + + /* Init HMAC/SHA224 out-of-order fields */ + state->hmac_sha_224_ooo.lens[0] = 0; + state->hmac_sha_224_ooo.lens[1] = 0; + state->hmac_sha_224_ooo.lens[2] = 0; + state->hmac_sha_224_ooo.lens[3] = 0; + state->hmac_sha_224_ooo.lens[4] = 0; + state->hmac_sha_224_ooo.lens[5] = 0; + state->hmac_sha_224_ooo.lens[6] = 0; + state->hmac_sha_224_ooo.lens[7] = 0; + state->hmac_sha_224_ooo.lens[8] = 0; + state->hmac_sha_224_ooo.lens[9] = 0; + state->hmac_sha_224_ooo.lens[10] = 0; + state->hmac_sha_224_ooo.lens[11] = 0; + state->hmac_sha_224_ooo.lens[12] = 0; + state->hmac_sha_224_ooo.lens[13] = 0; + state->hmac_sha_224_ooo.lens[14] = 0; + state->hmac_sha_224_ooo.lens[15] = 0; + state->hmac_sha_224_ooo.unused_lanes = 0xFEDCBA9876543210; + state->hmac_sha_224_ooo.num_lanes_inuse = 0; + /* sha256 and sha224 are very similar except for + * digest constants and output size + */ + for (j = 0; j < AVX512_NUM_SHA256_LANES; j++) { + state->hmac_sha_224_ooo.ldata[j].job_in_lane = NULL; + + p = state->hmac_sha_224_ooo.ldata[j].extra_block; + size = sizeof(state->hmac_sha_224_ooo.ldata[j].extra_block); + memset (p, 0x00, size); + p[64] = 0x80; + + p = state->hmac_sha_224_ooo.ldata[j].outer_block; + size = sizeof(state->hmac_sha_224_ooo.ldata[j].outer_block); + memset(p, 0x00, size); + p[7 * 4] = 0x80; /* digest 7 words long */ + p[64 - 2] = 0x02; /* length in little endian = 0x02E0 */ + p[64 - 1] = 0xE0; + } + + /* Init HMAC/SHA256 out-of-order fields */ + state->hmac_sha_256_ooo.lens[0] = 0; + state->hmac_sha_256_ooo.lens[1] = 0; + state->hmac_sha_256_ooo.lens[2] = 0; + state->hmac_sha_256_ooo.lens[3] = 0; + state->hmac_sha_256_ooo.lens[4] = 0; + state->hmac_sha_256_ooo.lens[5] = 0; + state->hmac_sha_256_ooo.lens[6] = 0; + state->hmac_sha_256_ooo.lens[7] = 0; + state->hmac_sha_256_ooo.lens[8] = 0; + state->hmac_sha_256_ooo.lens[9] = 0; + state->hmac_sha_256_ooo.lens[10] = 0; + state->hmac_sha_256_ooo.lens[11] = 0; + state->hmac_sha_256_ooo.lens[12] = 0; + state->hmac_sha_256_ooo.lens[13] = 0; + state->hmac_sha_256_ooo.lens[14] = 0; + state->hmac_sha_256_ooo.lens[15] = 0; + state->hmac_sha_256_ooo.unused_lanes = 0xFEDCBA9876543210; + state->hmac_sha_256_ooo.num_lanes_inuse = 0; + for (j = 0; j < AVX512_NUM_SHA256_LANES; j++) { + state->hmac_sha_256_ooo.ldata[j].job_in_lane = NULL; + state->hmac_sha_256_ooo.ldata[j].extra_block[64] = 0x80; + memset(state->hmac_sha_256_ooo.ldata[j].extra_block + 65, + 0x00, + 64 + 7); + /* hmac related */ + p = state->hmac_sha_256_ooo.ldata[j].outer_block; + memset(p + 8*4 + 1, + 0x00, + 64 - 8*4 - 1 - 2); + p[8 * 4] = 0x80; /* 8 digest words */ + p[64 - 2] = 0x03; /* length */ + p[64 - 1] = 0x00; + } + + /* Init HMAC/SHA384 out-of-order fields */ + state->hmac_sha_384_ooo.lens[0] = 0; + state->hmac_sha_384_ooo.lens[1] = 0; + state->hmac_sha_384_ooo.lens[2] = 0; + state->hmac_sha_384_ooo.lens[3] = 0; + state->hmac_sha_384_ooo.lens[4] = 0; + state->hmac_sha_384_ooo.lens[5] = 0; + state->hmac_sha_384_ooo.lens[6] = 0; + state->hmac_sha_384_ooo.lens[7] = 0; + state->hmac_sha_384_ooo.unused_lanes = 0xF76543210; + for (j = 0; j < AVX512_NUM_SHA512_LANES; j++) { + MB_MGR_HMAC_SHA_512_OOO *ctx = &state->hmac_sha_384_ooo; + + ctx->ldata[j].job_in_lane = NULL; + ctx->ldata[j].extra_block[SHA_384_BLOCK_SIZE] = 0x80; + memset(ctx->ldata[j].extra_block + (SHA_384_BLOCK_SIZE + 1), + 0x00, SHA_384_BLOCK_SIZE + 7); + p = ctx->ldata[j].outer_block; + /* special end point because this length is constant */ + memset(p + SHA384_DIGEST_SIZE_IN_BYTES + 1, 0x00, + SHA_384_BLOCK_SIZE - + SHA384_DIGEST_SIZE_IN_BYTES - 1 - 2); + /* mark the end */ + p[SHA384_DIGEST_SIZE_IN_BYTES] = 0x80; + /* hmac outer block length always of fixed size, + * it is OKey length, a whole message block length, 1024 bits, + * with padding plus the length of the inner digest, + * which is 384 bits, 1408 bits == 0x0580. + * The input message block needs to be converted to big endian + * within the sha implementation before use. + */ + p[SHA_384_BLOCK_SIZE - 2] = 0x05; + p[SHA_384_BLOCK_SIZE - 1] = 0x80; + } + + /* Init HMAC/SHA512 out-of-order fields */ + state->hmac_sha_512_ooo.lens[0] = 0; + state->hmac_sha_512_ooo.lens[1] = 0; + state->hmac_sha_512_ooo.lens[2] = 0; + state->hmac_sha_512_ooo.lens[3] = 0; + state->hmac_sha_512_ooo.lens[4] = 0; + state->hmac_sha_512_ooo.lens[5] = 0; + state->hmac_sha_512_ooo.lens[6] = 0; + state->hmac_sha_512_ooo.lens[7] = 0; + state->hmac_sha_512_ooo.unused_lanes = 0xF76543210; + for (j = 0; j < AVX512_NUM_SHA512_LANES; j++) { + MB_MGR_HMAC_SHA_512_OOO *ctx = &state->hmac_sha_512_ooo; + + ctx->ldata[j].job_in_lane = NULL; + ctx->ldata[j].extra_block[SHA_512_BLOCK_SIZE] = 0x80; + memset(ctx->ldata[j].extra_block + (SHA_512_BLOCK_SIZE + 1), + 0x00, SHA_512_BLOCK_SIZE + 7); + p = ctx->ldata[j].outer_block; + /* special end point because this length is constant */ + memset(p + SHA512_DIGEST_SIZE_IN_BYTES + 1, 0x00, + SHA_512_BLOCK_SIZE - + SHA512_DIGEST_SIZE_IN_BYTES - 1 - 2); + /* mark the end */ + p[SHA512_DIGEST_SIZE_IN_BYTES] = 0x80; + /* hmac outer block length always of fixed size, + * it is OKey length, a whole message block length, 1024 bits, + * with padding plus the length of the inner digest, + * which is 512 bits, 1536 bits == 0x600. + * The input message block needs to be converted to big endian + * within the sha implementation before use. + */ + p[SHA_512_BLOCK_SIZE - 2] = 0x06; + p[SHA_512_BLOCK_SIZE - 1] = 0x00; + } + + /* Init HMAC/MD5 out-of-order fields */ + state->hmac_md5_ooo.lens[0] = 0; + state->hmac_md5_ooo.lens[1] = 0; + state->hmac_md5_ooo.lens[2] = 0; + state->hmac_md5_ooo.lens[3] = 0; + state->hmac_md5_ooo.lens[4] = 0; + state->hmac_md5_ooo.lens[5] = 0; + state->hmac_md5_ooo.lens[6] = 0; + state->hmac_md5_ooo.lens[7] = 0; + state->hmac_md5_ooo.lens[8] = 0; + state->hmac_md5_ooo.lens[9] = 0; + state->hmac_md5_ooo.lens[10] = 0; + state->hmac_md5_ooo.lens[11] = 0; + state->hmac_md5_ooo.lens[12] = 0; + state->hmac_md5_ooo.lens[13] = 0; + state->hmac_md5_ooo.lens[14] = 0; + state->hmac_md5_ooo.lens[15] = 0; + state->hmac_md5_ooo.unused_lanes = 0xFEDCBA9876543210; + state->hmac_md5_ooo.num_lanes_inuse = 0; + for (j = 0; j < AVX512_NUM_MD5_LANES; j++) { + state->hmac_md5_ooo.ldata[j].job_in_lane = NULL; + + p = state->hmac_md5_ooo.ldata[j].extra_block; + size = sizeof(state->hmac_md5_ooo.ldata[j].extra_block); + memset (p, 0x00, size); + p[64] = 0x80; + + p = state->hmac_md5_ooo.ldata[j].outer_block; + size = sizeof(state->hmac_md5_ooo.ldata[j].outer_block); + memset(p, 0x00, size); + p[4 * 4] = 0x80; + p[64 - 7] = 0x02; + p[64 - 8] = 0x80; + } + + /* Init AES/XCBC OOO fields */ + state->aes_xcbc_ooo.lens[0] = 0; + state->aes_xcbc_ooo.lens[1] = 0; + state->aes_xcbc_ooo.lens[2] = 0; + state->aes_xcbc_ooo.lens[3] = 0; + state->aes_xcbc_ooo.lens[4] = 0; + state->aes_xcbc_ooo.lens[5] = 0; + state->aes_xcbc_ooo.lens[6] = 0; + state->aes_xcbc_ooo.lens[7] = 0; + state->aes_xcbc_ooo.unused_lanes = 0xF76543210; + for (j = 0; j < 8 ; j++) { + state->aes_xcbc_ooo.ldata[j].job_in_lane = NULL; + state->aes_xcbc_ooo.ldata[j].final_block[16] = 0x80; + memset(state->aes_xcbc_ooo.ldata[j].final_block + 17, 0x00, 15); + } + + /* Init AES-CCM auth out-of-order fields */ + for (j = 0; j < 8; j++) { + state->aes_ccm_ooo.init_done[j] = 0; + state->aes_ccm_ooo.lens[j] = 0; + state->aes_ccm_ooo.job_in_lane[j] = NULL; + } + state->aes_ccm_ooo.unused_lanes = 0xF76543210; + + /* Init AES-CMAC auth out-of-order fields */ + for (j = 0; j < 8; j++) { + state->aes_cmac_ooo.init_done[j] = 0; + state->aes_cmac_ooo.lens[j] = 0; + state->aes_cmac_ooo.job_in_lane[j] = NULL; + } + state->aes_cmac_ooo.unused_lanes = 0xF76543210; + + /* Init "in order" components */ + state->next_job = 0; + state->earliest_job = -1; + + /* set handlers */ + state->get_next_job = get_next_job_avx512; + state->submit_job = submit_job_avx512; + state->submit_job_nocheck = submit_job_nocheck_avx512; + state->get_completed_job = get_completed_job_avx512; + state->flush_job = flush_job_avx512; + state->queue_size = queue_size_avx512; + state->keyexp_128 = aes_keyexp_128_avx512; + state->keyexp_192 = aes_keyexp_192_avx512; + state->keyexp_256 = aes_keyexp_256_avx512; + state->cmac_subkey_gen_128 = aes_cmac_subkey_gen_avx512; + state->xcbc_keyexp = aes_xcbc_expand_key_avx512; + state->des_key_sched = des_key_schedule; + state->sha1_one_block = sha1_one_block_avx512; + state->sha1 = sha1_avx512; + state->sha224_one_block = sha224_one_block_avx512; + state->sha224 = sha224_avx512; + state->sha256_one_block = sha256_one_block_avx512; + state->sha256 = sha256_avx512; + state->sha384_one_block = sha384_one_block_avx512; + state->sha384 = sha384_avx512; + state->sha512_one_block = sha512_one_block_avx512; + state->sha512 = sha512_avx512; + state->md5_one_block = md5_one_block_avx512; + state->aes128_cfb_one = aes_cfb_128_one_avx512; + + state->eea3_1_buffer = zuc_eea3_1_buffer_avx; + state->eea3_4_buffer = zuc_eea3_4_buffer_avx; + state->eea3_n_buffer = zuc_eea3_n_buffer_avx; + state->eia3_1_buffer = zuc_eia3_1_buffer_avx; + + state->f8_1_buffer = kasumi_f8_1_buffer_avx; + state->f8_1_buffer_bit = kasumi_f8_1_buffer_bit_avx; + state->f8_2_buffer = kasumi_f8_2_buffer_avx; + state->f8_3_buffer = kasumi_f8_3_buffer_avx; + state->f8_4_buffer = kasumi_f8_4_buffer_avx; + state->f8_n_buffer = kasumi_f8_n_buffer_avx; + state->f9_1_buffer = kasumi_f9_1_buffer_avx; + state->f9_1_buffer_user = kasumi_f9_1_buffer_user_avx; + state->kasumi_init_f8_key_sched = kasumi_init_f8_key_sched_avx; + state->kasumi_init_f9_key_sched = kasumi_init_f9_key_sched_avx; + state->kasumi_key_sched_size = kasumi_key_sched_size_avx; + + state->snow3g_f8_1_buffer_bit = snow3g_f8_1_buffer_bit_avx2; + state->snow3g_f8_1_buffer = snow3g_f8_1_buffer_avx2; + state->snow3g_f8_2_buffer = snow3g_f8_2_buffer_avx2; + state->snow3g_f8_4_buffer = snow3g_f8_4_buffer_avx2; + state->snow3g_f8_8_buffer = snow3g_f8_8_buffer_avx2; + state->snow3g_f8_n_buffer = snow3g_f8_n_buffer_avx2; + state->snow3g_f8_8_buffer_multikey = snow3g_f8_8_buffer_multikey_avx2; + state->snow3g_f8_n_buffer_multikey = snow3g_f8_n_buffer_multikey_avx2; + state->snow3g_f9_1_buffer = snow3g_f9_1_buffer_avx2; + state->snow3g_init_key_sched = snow3g_init_key_sched_avx2; + state->snow3g_key_sched_size = snow3g_key_sched_size_avx2; + + if ((state->features & IMB_FEATURE_VAES) == IMB_FEATURE_VAES) { + submit_job_aes_cntr_avx512 = vaes_submit_cntr_avx512; + submit_job_aes_cntr_bit_avx512 = vaes_submit_cntr_bit_avx512; + } +#ifndef NO_GCM + if ((state->features & (IMB_FEATURE_VAES | IMB_FEATURE_VPCLMULQDQ)) == + (IMB_FEATURE_VAES | IMB_FEATURE_VPCLMULQDQ)) { + state->gcm128_enc = aes_gcm_enc_128_vaes_avx512; + state->gcm192_enc = aes_gcm_enc_192_vaes_avx512; + state->gcm256_enc = aes_gcm_enc_256_vaes_avx512; + state->gcm128_dec = aes_gcm_dec_128_vaes_avx512; + state->gcm192_dec = aes_gcm_dec_192_vaes_avx512; + state->gcm256_dec = aes_gcm_dec_256_vaes_avx512; + state->gcm128_init = aes_gcm_init_128_vaes_avx512; + state->gcm192_init = aes_gcm_init_192_vaes_avx512; + state->gcm256_init = aes_gcm_init_256_vaes_avx512; + state->gcm128_enc_update = aes_gcm_enc_128_update_vaes_avx512; + state->gcm192_enc_update = aes_gcm_enc_192_update_vaes_avx512; + state->gcm256_enc_update = aes_gcm_enc_256_update_vaes_avx512; + state->gcm128_dec_update = aes_gcm_dec_128_update_vaes_avx512; + state->gcm192_dec_update = aes_gcm_dec_192_update_vaes_avx512; + state->gcm256_dec_update = aes_gcm_dec_256_update_vaes_avx512; + state->gcm128_enc_finalize = + aes_gcm_enc_128_finalize_vaes_avx512; + state->gcm192_enc_finalize = + aes_gcm_enc_192_finalize_vaes_avx512; + state->gcm256_enc_finalize = + aes_gcm_enc_256_finalize_vaes_avx512; + state->gcm128_dec_finalize = + aes_gcm_dec_128_finalize_vaes_avx512; + state->gcm192_dec_finalize = + aes_gcm_dec_192_finalize_vaes_avx512; + state->gcm256_dec_finalize = + aes_gcm_dec_256_finalize_vaes_avx512; + state->gcm128_precomp = aes_gcm_precomp_128_vaes_avx512; + state->gcm192_precomp = aes_gcm_precomp_192_vaes_avx512; + state->gcm256_precomp = aes_gcm_precomp_256_vaes_avx512; + state->gcm128_pre = aes_gcm_pre_128_vaes_avx512; + state->gcm192_pre = aes_gcm_pre_192_vaes_avx512; + state->gcm256_pre = aes_gcm_pre_256_vaes_avx512; + + submit_job_aes_gcm_enc_avx512 = vaes_submit_gcm_enc_avx512; + submit_job_aes_gcm_dec_avx512 = vaes_submit_gcm_dec_avx512; + } else { + state->gcm128_enc = aes_gcm_enc_128_avx512; + state->gcm192_enc = aes_gcm_enc_192_avx512; + state->gcm256_enc = aes_gcm_enc_256_avx512; + state->gcm128_dec = aes_gcm_dec_128_avx512; + state->gcm192_dec = aes_gcm_dec_192_avx512; + state->gcm256_dec = aes_gcm_dec_256_avx512; + state->gcm128_init = aes_gcm_init_128_avx512; + state->gcm192_init = aes_gcm_init_192_avx512; + state->gcm256_init = aes_gcm_init_256_avx512; + state->gcm128_enc_update = aes_gcm_enc_128_update_avx512; + state->gcm192_enc_update = aes_gcm_enc_192_update_avx512; + state->gcm256_enc_update = aes_gcm_enc_256_update_avx512; + state->gcm128_dec_update = aes_gcm_dec_128_update_avx512; + state->gcm192_dec_update = aes_gcm_dec_192_update_avx512; + state->gcm256_dec_update = aes_gcm_dec_256_update_avx512; + state->gcm128_enc_finalize = aes_gcm_enc_128_finalize_avx512; + state->gcm192_enc_finalize = aes_gcm_enc_192_finalize_avx512; + state->gcm256_enc_finalize = aes_gcm_enc_256_finalize_avx512; + state->gcm128_dec_finalize = aes_gcm_dec_128_finalize_avx512; + state->gcm192_dec_finalize = aes_gcm_dec_192_finalize_avx512; + state->gcm256_dec_finalize = aes_gcm_dec_256_finalize_avx512; + state->gcm128_precomp = aes_gcm_precomp_128_avx512; + state->gcm192_precomp = aes_gcm_precomp_192_avx512; + state->gcm256_precomp = aes_gcm_precomp_256_avx512; + state->gcm128_pre = aes_gcm_pre_128_avx512; + state->gcm192_pre = aes_gcm_pre_192_avx512; + state->gcm256_pre = aes_gcm_pre_256_avx512; + } +#endif +} + +#include "mb_mgr_code.h" diff --git a/src/spdk/intel-ipsec-mb/avx512/mb_mgr_des_avx512.asm b/src/spdk/intel-ipsec-mb/avx512/mb_mgr_des_avx512.asm new file mode 100644 index 000000000..decea625b --- /dev/null +++ b/src/spdk/intel-ipsec-mb/avx512/mb_mgr_des_avx512.asm @@ -0,0 +1,524 @@ +;; +;; Copyright (c) 2017-2018, Intel Corporation +;; +;; Redistribution and use in source and binary forms, with or without +;; modification, are permitted provided that the following conditions are met: +;; +;; * Redistributions of source code must retain the above copyright notice, +;; this list of conditions and the following disclaimer. +;; * Redistributions in binary form must reproduce the above copyright +;; notice, this list of conditions and the following disclaimer in the +;; documentation and/or other materials provided with the distribution. +;; * Neither the name of Intel Corporation nor the names of its contributors +;; may be used to endorse or promote products derived from this software +;; without specific prior written permission. +;; +;; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +;; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +;; IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +;; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE +;; FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +;; DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +;; SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +;; CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +;; OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +;; + +;; In System V AMD64 ABI +;; calle saves: RBX, RBP, R12-R15 +;; Windows x64 ABI +;; calle saves: RBX, RBP, RDI, RSI, RSP, R12-R15 + +;; +;; Registers: RAX RBX RCX RDX RBP RSI RDI R8 R9 R10 R11 R12 R13 R14 R15 +;; ----------------------------------------------------------- +;; Windows clobbers: RAX R8 R9 R10 R11 +;; Windows preserves: RBX RCX RDX RBP RSI RDI R12 R13 R14 R15 +;; ----------------------------------------------------------- +;; Linux clobbers: RAX RCX RDX R10 R11 +;; Linux preserves: RBX RBP RSI RDI R8 R9 R12 R13 R14 R15 +;; ----------------------------------------------------------- +;; Clobbers ZMM0-31, K1-7 (K1-2 and K4-6 here but DES underneath clobbers K1-7). + +%include "include/os.asm" +%include "include/reg_sizes.asm" +%include "job_aes_hmac.asm" +%include "mb_mgr_datastruct.asm" +%include "constants.asm" +;%define DO_DBGPRINT +%include "include/dbgprint.asm" + +extern docsis_des_x16_enc_avx512 +extern docsis_des_x16_dec_avx512 +extern des_x16_cbc_enc_avx512 +extern des_x16_cbc_dec_avx512 +extern des3_x16_cbc_enc_avx512 +extern des3_x16_cbc_dec_avx512 + +%ifdef LINUX +%define arg1 rdi +%define arg2 rsi +%define arg3 rdx +%define arg4 rcx +%else +%define arg1 rcx +%define arg2 rdx +%define arg3 r8 +%define arg4 r9 +%endif + +%define STATE arg1 +%define JOB arg2 + +%define IA0 arg3 +%define IA1 arg4 +%define IA2 r10 + +%define MIN_IDX r11 +%define MIN_LEN rax +%define LANE r11 + +%define AVX512_NUM_DES_LANES 16 + +%define ZTMP0 zmm0 +%define ZTMP1 zmm1 +%define ZTMP2 zmm2 +%define ZTMP3 zmm3 +%define ZTMP4 zmm4 +%define ZTMP5 zmm5 +%define ZTMP6 zmm6 +%define ZTMP7 zmm7 +%define ZTMP8 zmm8 +%define ZTMP9 zmm9 + +;;; =========================================================================== +;;; =========================================================================== +;;; MACROS +;;; =========================================================================== +;;; =========================================================================== + +;;; =========================================================================== +;;; DES/DOCSIS DES job submit +;;; =========================================================================== +;;; DES_DOCSIS [in] - DES, DOCSIS or 3DES cipher selection +;;; ENC_DEC [in] - ENCrypt or DECrypt seection +%macro GENERIC_DES_SUBMIT 2 +%define %%DES_DOCSIS %1 +%define %%ENC_DEC %2 + + ;; get unsued lane and increment number of lanes in use + mov IA0, [STATE + _des_unused_lanes] + mov LANE, IA0 + and LANE, 0xF ;; just a nibble + shr IA0, 4 + mov [STATE + _des_unused_lanes], IA0 + add qword [STATE + _des_lanes_in_use], 1 + + ;; store job info in OOO structure + ;; - job pointer + mov [STATE + _des_job_in_lane + LANE*8], JOB + ;; - key schedule +%ifidn %%ENC_DEC, ENC + mov IA2, [JOB + _aes_enc_key_expanded] +%else + mov IA2, [JOB + _aes_dec_key_expanded] +%endif + mov [STATE + _des_args_keys + LANE*8], IA2 + ;; - IV + mov IA2, [JOB + _iv] + mov DWORD(IA0), [IA2] + mov DWORD(IA1), [IA2 + 4] + mov [STATE + _des_args_IV + LANE*4], DWORD(IA0) + mov [STATE + _des_args_IV + LANE*4 + (AVX512_NUM_DES_LANES*4)], DWORD(IA1) + ;; - src pointer + mov IA0, [JOB + _src] + add IA0, [JOB + _cipher_start_src_offset_in_bytes] + mov [STATE + _des_args_in + LANE*8], IA0 + ;; - destination pointer + mov IA1, [JOB + _dst] + mov [STATE + _des_args_out + LANE*8], IA1 + ;; - length in bytes (block aligned) + mov IA2, [JOB + _msg_len_to_cipher_in_bytes] + and IA2, -8 + mov [STATE + _des_lens + LANE*2], WORD(IA2) +%ifidn %%DES_DOCSIS, DOCSIS + ;; - block length + mov [STATE + _des_args_BLen + LANE*4], DWORD(IA2) + ;; - last in + add IA0, IA2 + mov [STATE + _des_args_LIn + LANE*8], IA0 + ;; - last out + add IA1, IA2 + mov [STATE + _des_args_LOut + LANE*8], IA1 + ;; - partial length + mov IA2, [JOB + _msg_len_to_cipher_in_bytes] + and IA2, 7 + mov [STATE + _des_args_PLen + LANE*4], DWORD(IA2) +%endif ; DOCSIS + ;; is there enough jobs to process them in parallel? + cmp qword [STATE + _des_lanes_in_use], AVX512_NUM_DES_LANES + jb %%_des_submit_null_end + ;; schedule the processing + ;; - find min job size + vmovdqa XWORD(ZTMP0), [STATE + _des_lens + 2*0] + vphminposuw XWORD(ZTMP2), XWORD(ZTMP0) + vpextrw DWORD(MIN_LEN), XWORD(ZTMP2), 0 ; min value + vpextrw DWORD(MIN_IDX), XWORD(ZTMP2), 1 ; min index + vmovdqa XWORD(ZTMP1), [STATE + _des_lens + 2*8] + vphminposuw XWORD(ZTMP2), XWORD(ZTMP1) + vpextrw DWORD(IA2), XWORD(ZTMP2), 0 ; min value + cmp DWORD(MIN_LEN), DWORD(IA2) + jle %%_use_min + vpextrw DWORD(MIN_IDX), XWORD(ZTMP2), 1 ; min index + add DWORD(MIN_IDX), 8 ; but index +8 + mov MIN_LEN, IA2 ; min len +%%_use_min: + cmp MIN_LEN, 0 + je %%_len_is_0 + + vpbroadcastw XWORD(ZTMP3), WORD(MIN_LEN) + vpsubw XWORD(ZTMP0), XWORD(ZTMP0), XWORD(ZTMP3) + vmovdqa [STATE + _des_lens + 2*0], XWORD(ZTMP0) + vpsubw XWORD(ZTMP1), XWORD(ZTMP1), XWORD(ZTMP3) + vmovdqa [STATE + _des_lens + 2*8], XWORD(ZTMP1) + + push MIN_IDX + mov arg2, MIN_LEN +%ifidn %%ENC_DEC, ENC + ;; encrypt +%ifidn %%DES_DOCSIS, DOCSIS + call docsis_des_x16_enc_avx512 +%endif +%ifidn %%DES_DOCSIS, DES + call des_x16_cbc_enc_avx512 +%endif +%ifidn %%DES_DOCSIS, 3DES + call des3_x16_cbc_enc_avx512 +%endif +%else ; ENC + ;; decrypt +%ifidn %%DES_DOCSIS, DOCSIS + call docsis_des_x16_dec_avx512 +%endif +%ifidn %%DES_DOCSIS, DES + call des_x16_cbc_dec_avx512 +%endif +%ifidn %%DES_DOCSIS, 3DES + call des3_x16_cbc_dec_avx512 +%endif +%endif ; DEC + pop MIN_IDX + jmp %%_des_submit_end + +%%_des_submit_null_end: + xor rax, rax + jmp %%_des_submit_return + +%%_len_is_0: +%ifidn %%DES_DOCSIS, DOCSIS + cmp dword [STATE + _des_args_PLen + MIN_IDX*4], 0 + jz %%_des_submit_end + push MIN_IDX + xor arg2, arg2 ; len is 0 +%ifidn %%ENC_DEC, ENC + call docsis_des_x16_enc_avx512 +%else ; ENC + call docsis_des_x16_dec_avx512 +%endif ; DEC + pop MIN_IDX +%endif ; DOCSIS + ;; fall trough +%%_des_submit_end: + ;; return a job + ;; - decrement number of jobs in use + sub qword [STATE + _des_lanes_in_use], 1 + ;; - put the lane back to free lanes pool + mov IA0, [STATE + _des_unused_lanes] + shl IA0, 4 + or IA0, MIN_IDX + mov [STATE + _des_unused_lanes], IA0 + ;; - mark job as complete + ;; - clear job pointer + mov rax, [STATE + _des_job_in_lane + MIN_IDX*8] + mov qword [STATE + _des_job_in_lane + MIN_IDX*8], 0 + or dword [rax + _status], STS_COMPLETED_AES + +%ifdef SAFE_DATA + ;; Clear IV + mov dword [STATE + _des_args_IV + MIN_IDX*4], 0 + mov dword [STATE + _des_args_IV + MIN_IDX*4 + (AVX512_NUM_DES_LANES*4)], 0 +%endif + vzeroupper +%%_des_submit_return: +%endmacro + +;;; =========================================================================== +;;; DES/DOCSIS DES flush +;;; =========================================================================== +;;; DES_DOCSIS [in] - DES, DOCSIS or 3DES cipher selection +;;; ENC_DEC [in] - ENCrypt or DECrypt selection +;;; +;;; Clobbers k1, k2, k4, k5 and k6 +%macro GENERIC_DES_FLUSH 2 +%define %%DES_DOCSIS %1 +%define %%ENC_DEC %2 + + cmp qword [STATE + _des_lanes_in_use], 0 + je %%_des_flush_null_end + + ;; find non-null job + vpxord ZTMP0, ZTMP0, ZTMP0 + vmovdqu64 ZTMP1, [STATE + _des_job_in_lane + (0*PTR_SZ)] + vmovdqu64 ZTMP2, [STATE + _des_job_in_lane + (8*PTR_SZ)] + vpcmpq k1, ZTMP1, ZTMP0, 4 ; NEQ + vpcmpq k2, ZTMP2, ZTMP0, 4 ; NEQ + xor IA0, IA0 + xor IA1, IA1 + kmovw DWORD(IA0), k1 + kmovw DWORD(IA1), k2 + mov DWORD(IA2), DWORD(IA1) + shl DWORD(IA2), 8 + or DWORD(IA2), DWORD(IA0) ; mask of non-null jobs in IA2 + not BYTE(IA0) + kmovw k4, DWORD(IA0) + not BYTE(IA1) + kmovw k5, DWORD(IA1) + mov DWORD(IA0), DWORD(IA2) + not WORD(IA0) + kmovw k6, DWORD(IA0) ; mask of NULL jobs in k4, k5 and k6 + mov DWORD(IA0), DWORD(IA2) + xor IA2, IA2 + bsf WORD(IA2), WORD(IA0) ; index of the 1st set bit in IA2 + + ;; copy good lane data into NULL lanes + ;; - k1(L8)/k2(H8) - masks of non-null jobs + ;; - k4(L8)/k5(H8)/k6 - masks of NULL jobs + ;; - IA2 index of 1st non-null job + + ;; - in pointer + mov IA0, [STATE + _des_args_in + IA2*8] + vpbroadcastq ZTMP1, IA0 + vmovdqu64 [STATE + _des_args_in + (0*PTR_SZ)]{k4}, ZTMP1 + vmovdqu64 [STATE + _des_args_in + (8*PTR_SZ)]{k5}, ZTMP1 + ;; - out pointer + mov IA0, [STATE + _des_args_out + IA2*8] + vpbroadcastq ZTMP1, IA0 + vmovdqu64 [STATE + _des_args_out + (0*PTR_SZ)]{k4}, ZTMP1 + vmovdqu64 [STATE + _des_args_out + (8*PTR_SZ)]{k5}, ZTMP1 + ;; - key schedule + mov IA0, [STATE + _des_args_keys + IA2*8] + vpbroadcastq ZTMP1, IA0 + vmovdqu64 [STATE + _des_args_keys + (0*PTR_SZ)]{k4}, ZTMP1 + vmovdqu64 [STATE + _des_args_keys + (8*PTR_SZ)]{k5}, ZTMP1 + ;; - zero partial len + vmovdqu32 [STATE + _des_args_PLen]{k6}, ZTMP0 + ;; - set len to UINT16_MAX + mov WORD(IA0), 0xffff + vpbroadcastw ZTMP1, WORD(IA0) + vmovdqu16 [STATE + _des_lens]{k6}, ZTMP1 + + ;; - IV + mov DWORD(IA0), [STATE + _des_args_IV + IA2*4] + mov DWORD(IA1), [STATE + _des_args_IV + IA2*4 + (16*4)] + vpbroadcastd ZTMP1, DWORD(IA0) + vpbroadcastd ZTMP2, DWORD(IA1) + vmovdqu32 [STATE + _des_args_IV]{k6}, ZTMP1 + vmovdqu32 [STATE + _des_args_IV + (16*4)]{k6}, ZTMP2 + + ;; schedule the processing + ;; - find min job size + vmovdqa XWORD(ZTMP0), [STATE + _des_lens + 2*0] + vphminposuw XWORD(ZTMP2), XWORD(ZTMP0) + vpextrw DWORD(MIN_LEN), XWORD(ZTMP2), 0 ; min value + vpextrw DWORD(MIN_IDX), XWORD(ZTMP2), 1 ; min index + vmovdqa XWORD(ZTMP1), [STATE + _des_lens + 2*8] + vphminposuw XWORD(ZTMP2), XWORD(ZTMP1) + vpextrw DWORD(IA2), XWORD(ZTMP2), 0 ; min value + cmp DWORD(MIN_LEN), DWORD(IA2) + jle %%_use_min + vpextrw DWORD(MIN_IDX), XWORD(ZTMP2), 1 ; min index + add DWORD(MIN_IDX), 8 ; but index +8 + mov MIN_LEN, IA2 ; min len +%%_use_min: + vpbroadcastw XWORD(ZTMP3), WORD(MIN_LEN) + vpsubw XWORD(ZTMP0), XWORD(ZTMP0), XWORD(ZTMP3) + vmovdqa [STATE + _des_lens + 2*0], XWORD(ZTMP0) + vpsubw XWORD(ZTMP1), XWORD(ZTMP1), XWORD(ZTMP3) + vmovdqa [STATE + _des_lens + 2*8], XWORD(ZTMP1) + + push MIN_IDX +%ifdef SAFE_DATA + ;; Save k6, which may be clobbered by following functions + kmovq IA0, k6 + push IA0 +%endif + + mov arg2, MIN_LEN +%ifidn %%ENC_DEC, ENC + ;; encrypt +%ifidn %%DES_DOCSIS, DOCSIS + call docsis_des_x16_enc_avx512 +%endif +%ifidn %%DES_DOCSIS, DES + call des_x16_cbc_enc_avx512 +%endif +%ifidn %%DES_DOCSIS, 3DES + call des3_x16_cbc_enc_avx512 +%endif +%else ; ENC + ;; decrypt +%ifidn %%DES_DOCSIS, DOCSIS + call docsis_des_x16_dec_avx512 +%endif +%ifidn %%DES_DOCSIS, DES + call des_x16_cbc_dec_avx512 +%endif +%ifidn %%DES_DOCSIS, 3DES + call des3_x16_cbc_dec_avx512 +%endif +%endif ; DEC +%ifdef SAFE_DATA + ;; Restore k6, which may have been clobbered by previous functions + pop IA0 + kmovq k6, IA0 +%endif + pop MIN_IDX + jmp %%_des_flush_end + +%%_des_flush_null_end: + xor rax, rax + jmp %%_des_flush_return +%%_des_flush_end: + ;; return a job + ;; - decrement number of jobs in use + sub qword [STATE + _des_lanes_in_use], 1 + ;; - put the lane back to free lanes pool + mov IA0, [STATE + _des_unused_lanes] + shl IA0, 4 + or IA0, MIN_IDX + mov [STATE + _des_unused_lanes], IA0 + ;; - mark job as complete + mov rax, [STATE + _des_job_in_lane + MIN_IDX*8] + or dword [rax + _status], STS_COMPLETED_AES + ;; - clear job pointer + mov qword [STATE + _des_job_in_lane + MIN_IDX*8], 0 +%ifdef SAFE_DATA + ; Set bit of lane of returned job + xor DWORD(IA0), DWORD(IA0) + bts DWORD(IA0), DWORD(MIN_IDX) + kmovd k1, DWORD(IA0) + kord k6, k1, k6 + + ;; Clear IV of returned job and "NULL lanes" (k6 contains the mask of the jobs) + vpxorq ZTMP1, ZTMP1 + vmovdqa32 [STATE + _des_args_IV]{k6}, ZTMP1 + vmovdqa32 [STATE + _des_args_IV + (16*4)]{k6}, ZTMP1 +%endif +%%_des_flush_return: + vzeroupper +%endmacro + +;;; ======================================================== +;;; DATA + +section .data +default rel + +;;; ======================================================== +;;; CODE +section .text + +;;; arg 1 : pointer to DES OOO structure +;;; arg 2 : job +align 64 +MKGLOBAL(submit_job_des_cbc_enc_avx512,function,internal) +submit_job_des_cbc_enc_avx512: + GENERIC_DES_SUBMIT DES, ENC + ret + +;;; arg 1 : pointer to DES OOO structure +;;; arg 2 : job +align 64 +MKGLOBAL(submit_job_des_cbc_dec_avx512,function,internal) +submit_job_des_cbc_dec_avx512: + GENERIC_DES_SUBMIT DES, DEC + ret + +;;; arg 1 : pointer to DES OOO structure +;;; arg 2 : job +align 64 +MKGLOBAL(submit_job_docsis_des_enc_avx512,function,internal) +submit_job_docsis_des_enc_avx512: + GENERIC_DES_SUBMIT DOCSIS, ENC + ret + +;;; arg 1 : pointer to DES OOO structure +;;; arg 2 : job +align 64 +MKGLOBAL(submit_job_docsis_des_dec_avx512,function,internal) +submit_job_docsis_des_dec_avx512: + GENERIC_DES_SUBMIT DOCSIS, DEC + ret + +;;; arg 1 : pointer to DES OOO structure +;;; arg 2 : job +align 64 +MKGLOBAL(submit_job_3des_cbc_enc_avx512,function,internal) +submit_job_3des_cbc_enc_avx512: + GENERIC_DES_SUBMIT 3DES, ENC + ret + +;;; arg 1 : pointer to DES OOO structure +;;; arg 2 : job +align 64 +MKGLOBAL(submit_job_3des_cbc_dec_avx512,function,internal) +submit_job_3des_cbc_dec_avx512: + GENERIC_DES_SUBMIT 3DES, DEC + ret + +;;; arg 1 : pointer to DES OOO structure +align 64 +MKGLOBAL(flush_job_des_cbc_enc_avx512,function,internal) +flush_job_des_cbc_enc_avx512: + GENERIC_DES_FLUSH DES, ENC + ret + +;;; arg 1 : pointer to DES OOO structure +align 64 +MKGLOBAL(flush_job_des_cbc_dec_avx512,function,internal) +flush_job_des_cbc_dec_avx512: + GENERIC_DES_FLUSH DES, DEC + ret + +;;; arg 1 : pointer to DES OOO structure +align 64 +MKGLOBAL(flush_job_docsis_des_enc_avx512,function,internal) +flush_job_docsis_des_enc_avx512: + GENERIC_DES_FLUSH DOCSIS, ENC + ret + +;;; arg 1 : pointer to DES OOO structure +align 64 +MKGLOBAL(flush_job_docsis_des_dec_avx512,function,internal) +flush_job_docsis_des_dec_avx512: + GENERIC_DES_FLUSH DOCSIS, DEC + ret + +;;; arg 1 : pointer to DES OOO structure +align 64 +MKGLOBAL(flush_job_3des_cbc_enc_avx512,function,internal) +flush_job_3des_cbc_enc_avx512: + GENERIC_DES_FLUSH 3DES, ENC + ret + +;;; arg 1 : pointer to DES OOO structure +align 64 +MKGLOBAL(flush_job_3des_cbc_dec_avx512,function,internal) +flush_job_3des_cbc_dec_avx512: + GENERIC_DES_FLUSH 3DES, DEC + ret + +%ifdef LINUX +section .note.GNU-stack noalloc noexec nowrite progbits +%endif diff --git a/src/spdk/intel-ipsec-mb/avx512/mb_mgr_hmac_flush_avx512.asm b/src/spdk/intel-ipsec-mb/avx512/mb_mgr_hmac_flush_avx512.asm new file mode 100644 index 000000000..5fa08053f --- /dev/null +++ b/src/spdk/intel-ipsec-mb/avx512/mb_mgr_hmac_flush_avx512.asm @@ -0,0 +1,367 @@ +;; +;; Copyright (c) 2012-2018, Intel Corporation +;; +;; Redistribution and use in source and binary forms, with or without +;; modification, are permitted provided that the following conditions are met: +;; +;; * Redistributions of source code must retain the above copyright notice, +;; this list of conditions and the following disclaimer. +;; * Redistributions in binary form must reproduce the above copyright +;; notice, this list of conditions and the following disclaimer in the +;; documentation and/or other materials provided with the distribution. +;; * Neither the name of Intel Corporation nor the names of its contributors +;; may be used to endorse or promote products derived from this software +;; without specific prior written permission. +;; +;; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +;; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +;; IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +;; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE +;; FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +;; DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +;; SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +;; CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +;; OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +;; + +;; In System V AMD64 ABI +;; calle saves: RBX, RBP, R12-R15 +;; Windows x64 ABI +;; calle saves: RBX, RBP, RDI, RSI, RSP, R12-R15 +;; +;; Registers: RAX RBX RCX RDX RBP RSI RDI R8 R9 R10 R11 R12 R13 R14 R15 +;; ----------------------------------------------------------- +;; Windows clobbers: RAX RCX RDX R8 R9 R10 R11 +;; Windows preserves: RBX RBP RSI RDI R12 R13 R14 R15 +;; ----------------------------------------------------------- +;; Linux clobbers: RAX RSI RDI R8 R9 R10 R11 +;; Linux preserves: RBX RCX RDX RBP R12 R13 R14 R15 +;; ----------------------------------------------------------- +;; Clobbers ZMM0-31 + +%include "include/os.asm" +%include "job_aes_hmac.asm" +%include "mb_mgr_datastruct.asm" +%include "include/reg_sizes.asm" + +;; %define DO_DBGPRINT +%include "include/dbgprint.asm" + +extern sha1_x16_avx512 + +section .data +default rel + +align 16 +byteswap: + dq 0x0405060700010203 + dq 0x0c0d0e0f08090a0b + +align 32 +len_masks: + dq 0x000000000000FFFF, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000 + dq 0x00000000FFFF0000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000 + dq 0x0000FFFF00000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000 + dq 0xFFFF000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000 + dq 0x0000000000000000, 0x000000000000FFFF, 0x0000000000000000, 0x0000000000000000 + dq 0x0000000000000000, 0x00000000FFFF0000, 0x0000000000000000, 0x0000000000000000 + dq 0x0000000000000000, 0x0000FFFF00000000, 0x0000000000000000, 0x0000000000000000 + dq 0x0000000000000000, 0xFFFF000000000000, 0x0000000000000000, 0x0000000000000000 + dq 0x0000000000000000, 0x0000000000000000, 0x000000000000FFFF, 0x0000000000000000 + dq 0x0000000000000000, 0x0000000000000000, 0x00000000FFFF0000, 0x0000000000000000 + dq 0x0000000000000000, 0x0000000000000000, 0x0000FFFF00000000, 0x0000000000000000 + dq 0x0000000000000000, 0x0000000000000000, 0xFFFF000000000000, 0x0000000000000000 + dq 0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x000000000000FFFF + dq 0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x00000000FFFF0000 + dq 0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000FFFF00000000 + dq 0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0xFFFF000000000000 + +lane_1: dq 1 +lane_2: dq 2 +lane_3: dq 3 +lane_4: dq 4 +lane_5: dq 5 +lane_6: dq 6 +lane_7: dq 7 +lane_8: dq 8 +lane_9: dq 9 +lane_10: dq 10 +lane_11: dq 11 +lane_12: dq 12 +lane_13: dq 13 +lane_14: dq 14 +lane_15: dq 15 + +section .text + +%if 1 +%ifdef LINUX +%define arg1 rdi +%define arg2 rsi +%else +%define arg1 rcx +%define arg2 rdx +%endif + +%define state arg1 +%define job arg2 +%define len2 arg2 + +; idx needs to be in rbx, rdi, rbp +%define idx rbp + +%define unused_lanes r9 +%define lane_data r9 +%define tmp2 r9 +%define num_lanes_inuse r12 +%define len_upper r13 +%define idx_upper r14 + +%define job_rax rax +%define tmp1 rax +%define size_offset rax +%define tmp rax +%define start_offset rax + +%define tmp3 arg1 + +%define extra_blocks arg2 +%define p arg2 + +%define tmp4 r8 + +%endif + +; we clobber rbp, called routine clobbers r12-r15 +struc STACK +_gpr_save: resq 5 +_rsp_save: resq 1 +endstruc + +%define APPEND(a,b) a %+ b + +; JOB* flush_job_hmac_avx(MB_MGR_HMAC_SHA_1_OOO *state) +; arg 1 : rcx : state +MKGLOBAL(flush_job_hmac_avx512,function,internal) +flush_job_hmac_avx512: + + mov rax, rsp + sub rsp, STACK_size + and rsp, -32 ; align stack to 32 byte boundary + mov [rsp + _gpr_save + 8*0], rbp + mov [rsp + _gpr_save + 8*1], r12 + mov [rsp + _gpr_save + 8*2], r13 + mov [rsp + _gpr_save + 8*3], r14 + mov [rsp + _gpr_save + 8*4], r15 + mov [rsp + _rsp_save], rax + + DBGPRINTL "---------- start hmac flush avx512 -----------" + + mov DWORD(num_lanes_inuse), [state + _num_lanes_inuse_sha1] ;empty? + cmp num_lanes_inuse, 0 + jz return_null + + ; find a lane with a non-null job + xor idx, idx +%assign I 1 +%rep 15 + cmp qword [state + _ldata + (I * _HMAC_SHA1_LANE_DATA_size) + _job_in_lane], 0 + cmovne idx, [rel APPEND(lane_,I)] +%assign I (I+1) +%endrep + +copy_lane_data: + ; copy valid lane (idx) to empty lanes + vmovdqa ymm0, [state + _lens] + mov tmp, [state + _args_data_ptr + PTR_SZ*idx] + +%assign I 0 +%rep 16 + cmp qword [state + _ldata + I * _HMAC_SHA1_LANE_DATA_size + _job_in_lane], 0 + jne APPEND(skip_,I) + mov [state + _args_data_ptr + PTR_SZ*I], tmp + vpor ymm0, ymm0, [rel len_masks + 32*I] ; 32 for ymm, 16 for xmm +APPEND(skip_,I): +%assign I (I+1) +%endrep + vmovdqa [state + _lens], ymm0 + + vphminposuw xmm1, xmm0 + vpextrw DWORD(len2), xmm1, 0 ; min value + vpextrw DWORD(idx), xmm1, 1 ; min index (0...7) + + vmovdqa xmm2, [state + _lens + 8*2] + vphminposuw xmm3, xmm2 + vpextrw DWORD(len_upper), xmm3, 0 ; min value + vpextrw DWORD(idx_upper), xmm3, 1 ; min index (8...F) + + cmp len2, len_upper + jle use_min + + vmovdqa xmm1, xmm3 + mov len2, len_upper + mov idx, idx_upper ; idx would be in range 0..7 + add idx, 8 ; to reflect that index is in 8..F range + +use_min: + DBGPRINTL64 "FLUSH min_length", len2 + DBGPRINTL64 "FLUSH min_length index ", idx + cmp len2, 0 + je len_is_0 + + vpbroadcastw xmm1, xmm1 + DBGPRINTL_XMM "FLUSH lens after shuffle", xmm1 + + vpsubw xmm0, xmm0, xmm1 + vmovdqa [state + _lens], xmm0 + vpsubw xmm2, xmm2, xmm1 + vmovdqa [state + _lens + 8*2], xmm2 + DBGPRINTL_XMM "FLUSH lens immediately after min subtraction (0..7)", xmm0 + DBGPRINTL_XMM "FLUSH lens immediately after min subtraction (8..F)", xmm2 + + ; "state" and "args" are the same address, arg1 + ; len is arg2 + call sha1_x16_avx512 + ; state and idx are intact + +len_is_0: + ; process completed job "idx" + imul lane_data, idx, _HMAC_SHA1_LANE_DATA_size + lea lane_data, [state + _ldata + lane_data] + mov DWORD(extra_blocks), [lane_data + _extra_blocks] + cmp extra_blocks, 0 + jne proc_extra_blocks + cmp dword [lane_data + _outer_done], 0 + jne end_loop + +proc_outer: + mov dword [lane_data + _outer_done], 1 + mov DWORD(size_offset), [lane_data + _size_offset] + mov qword [lane_data + _extra_block + size_offset], 0 + mov word [state + _lens + 2*idx], 1 + lea tmp, [lane_data + _outer_block] + mov job, [lane_data + _job_in_lane] + mov [state + _args_data_ptr + PTR_SZ*idx], tmp + + vmovd xmm0, [state + _args_digest + SHA1_DIGEST_WORD_SIZE*idx + 0*SHA1_DIGEST_ROW_SIZE] + vpinsrd xmm0, xmm0, [state + _args_digest + SHA1_DIGEST_WORD_SIZE*idx + 1*SHA1_DIGEST_ROW_SIZE], 1 + vpinsrd xmm0, xmm0, [state + _args_digest + SHA1_DIGEST_WORD_SIZE*idx + 2*SHA1_DIGEST_ROW_SIZE], 2 + vpinsrd xmm0, xmm0, [state + _args_digest + SHA1_DIGEST_WORD_SIZE*idx + 3*SHA1_DIGEST_ROW_SIZE], 3 + vpshufb xmm0, xmm0, [rel byteswap] + mov DWORD(tmp), [state + _args_digest + SHA1_DIGEST_WORD_SIZE*idx + 4*SHA1_DIGEST_ROW_SIZE] + bswap DWORD(tmp) + vmovdqa [lane_data + _outer_block], xmm0 + mov [lane_data + _outer_block + 4*4], DWORD(tmp) + + mov tmp, [job + _auth_key_xor_opad] + vmovdqu xmm0, [tmp] + mov DWORD(tmp), [tmp + 4*4] + vmovd [state + _args_digest + SHA1_DIGEST_WORD_SIZE*idx + 0*SHA1_DIGEST_ROW_SIZE], xmm0 + vpextrd [state + _args_digest + SHA1_DIGEST_WORD_SIZE*idx + 1*SHA1_DIGEST_ROW_SIZE], xmm0, 1 + vpextrd [state + _args_digest + SHA1_DIGEST_WORD_SIZE*idx + 2*SHA1_DIGEST_ROW_SIZE], xmm0, 2 + vpextrd [state + _args_digest + SHA1_DIGEST_WORD_SIZE*idx + 3*SHA1_DIGEST_ROW_SIZE], xmm0, 3 + mov [state + _args_digest + SHA1_DIGEST_WORD_SIZE*idx + 4*SHA1_DIGEST_ROW_SIZE], DWORD(tmp) + jmp copy_lane_data + + align 16 +proc_extra_blocks: + mov DWORD(start_offset), [lane_data + _start_offset] + mov [state + _lens + 2*idx], WORD(extra_blocks) + lea tmp, [lane_data + _extra_block + start_offset] + mov [state + _args_data_ptr + PTR_SZ*idx], tmp + mov dword [lane_data + _extra_blocks], 0 + jmp copy_lane_data + +return_null: + DBGPRINTL "FLUSH *** ---------- return null" + xor job_rax, job_rax + jmp return + + align 16 +end_loop: + mov job_rax, [lane_data + _job_in_lane] + mov qword [lane_data + _job_in_lane], 0 + or dword [job_rax + _status], STS_COMPLETED_HMAC + + mov unused_lanes, [state + _unused_lanes] + shl unused_lanes, 4 ;; a nibble + or unused_lanes, idx + mov [state + _unused_lanes], unused_lanes + + sub dword [state + _num_lanes_inuse_sha1], 1 + + mov p, [job_rax + _auth_tag_output] + + ; copy 12 bytes + mov DWORD(tmp2), [state + _args_digest + SHA1_DIGEST_WORD_SIZE*idx + 0*SHA1_DIGEST_ROW_SIZE] + mov DWORD(tmp4), [state + _args_digest + SHA1_DIGEST_WORD_SIZE*idx + 1*SHA1_DIGEST_ROW_SIZE] + mov DWORD(r12), [state + _args_digest + SHA1_DIGEST_WORD_SIZE*idx + 2*SHA1_DIGEST_ROW_SIZE] + bswap DWORD(tmp2) + bswap DWORD(tmp4) + bswap DWORD(r12) + mov [p + 0*4], DWORD(tmp2) + mov [p + 1*4], DWORD(tmp4) + mov [p + 2*4], DWORD(r12) + + cmp qword [job_rax + _auth_tag_output_len_in_bytes], 12 + je clear_ret + + ;; copy remaining 8 bytes to return 20 byte digest + mov DWORD(r13), [state + _args_digest + SHA1_DIGEST_WORD_SIZE*idx + 3*SHA1_DIGEST_ROW_SIZE] + mov DWORD(r14), [state + _args_digest + SHA1_DIGEST_WORD_SIZE*idx + 4*SHA1_DIGEST_ROW_SIZE] + bswap DWORD(r13) + bswap DWORD(r14) + mov [p + 3*SHA1_DIGEST_WORD_SIZE], DWORD(r13) + mov [p + 4*SHA1_DIGEST_WORD_SIZE], DWORD(r14) + +clear_ret: + +%ifdef SAFE_DATA + vpxorq zmm0, zmm0 + + ;; Clear digest (20B), outer_block (20B) and extra_block (64B) + ;; of returned job and NULL jobs +%assign I 0 +%rep 16 + cmp qword [state + _ldata + (I*_HMAC_SHA1_LANE_DATA_size) + _job_in_lane], 0 + jne APPEND(skip_clear_,I) + + ;; Clear digest + mov dword [state + _args_digest + SHA1_DIGEST_WORD_SIZE*I + 0*SHA1_DIGEST_ROW_SIZE], 0 + mov dword [state + _args_digest + SHA1_DIGEST_WORD_SIZE*I + 1*SHA1_DIGEST_ROW_SIZE], 0 + mov dword [state + _args_digest + SHA1_DIGEST_WORD_SIZE*I + 2*SHA1_DIGEST_ROW_SIZE], 0 + mov dword [state + _args_digest + SHA1_DIGEST_WORD_SIZE*I + 3*SHA1_DIGEST_ROW_SIZE], 0 + mov dword [state + _args_digest + SHA1_DIGEST_WORD_SIZE*I + 4*SHA1_DIGEST_ROW_SIZE], 0 + + lea lane_data, [state + _ldata + (I*_HMAC_SHA1_LANE_DATA_size)] + + ;; Clear first 64 bytes of extra_block + vmovdqu64 [lane_data + _extra_block], zmm0 + + ;; Clear first 20 bytes of outer_block + vmovdqu64 [lane_data + _outer_block], xmm0 + mov dword [lane_data + _outer_block + 16], 0 + +APPEND(skip_clear_,I): +%assign I (I+1) +%endrep + +%endif ;; SAFE_DATA + +return: + DBGPRINTL "---------- exit hmac flush avx512 -----------" + vzeroupper + + mov rbp, [rsp + _gpr_save + 8*0] + mov r12, [rsp + _gpr_save + 8*1] + mov r13, [rsp + _gpr_save + 8*2] + mov r14, [rsp + _gpr_save + 8*3] + mov r15, [rsp + _gpr_save + 8*4] + mov rsp, [rsp + _rsp_save] + ret + + +%ifdef LINUX +section .note.GNU-stack noalloc noexec nowrite progbits +%endif diff --git a/src/spdk/intel-ipsec-mb/avx512/mb_mgr_hmac_sha_224_flush_avx512.asm b/src/spdk/intel-ipsec-mb/avx512/mb_mgr_hmac_sha_224_flush_avx512.asm new file mode 100644 index 000000000..656e854d5 --- /dev/null +++ b/src/spdk/intel-ipsec-mb/avx512/mb_mgr_hmac_sha_224_flush_avx512.asm @@ -0,0 +1,28 @@ +;; +;; Copyright (c) 2017-2018, Intel Corporation +;; +;; Redistribution and use in source and binary forms, with or without +;; modification, are permitted provided that the following conditions are met: +;; +;; * Redistributions of source code must retain the above copyright notice, +;; this list of conditions and the following disclaimer. +;; * Redistributions in binary form must reproduce the above copyright +;; notice, this list of conditions and the following disclaimer in the +;; documentation and/or other materials provided with the distribution. +;; * Neither the name of Intel Corporation nor the names of its contributors +;; may be used to endorse or promote products derived from this software +;; without specific prior written permission. +;; +;; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +;; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +;; IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +;; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE +;; FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +;; DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +;; SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +;; CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +;; OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +;; +%define SHA224 +%include "avx512/mb_mgr_hmac_sha_256_flush_avx512.asm" diff --git a/src/spdk/intel-ipsec-mb/avx512/mb_mgr_hmac_sha_224_submit_avx512.asm b/src/spdk/intel-ipsec-mb/avx512/mb_mgr_hmac_sha_224_submit_avx512.asm new file mode 100644 index 000000000..60a98918a --- /dev/null +++ b/src/spdk/intel-ipsec-mb/avx512/mb_mgr_hmac_sha_224_submit_avx512.asm @@ -0,0 +1,28 @@ +;; +;; Copyright (c) 2017-2018, Intel Corporation +;; +;; Redistribution and use in source and binary forms, with or without +;; modification, are permitted provided that the following conditions are met: +;; +;; * Redistributions of source code must retain the above copyright notice, +;; this list of conditions and the following disclaimer. +;; * Redistributions in binary form must reproduce the above copyright +;; notice, this list of conditions and the following disclaimer in the +;; documentation and/or other materials provided with the distribution. +;; * Neither the name of Intel Corporation nor the names of its contributors +;; may be used to endorse or promote products derived from this software +;; without specific prior written permission. +;; +;; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +;; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +;; IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +;; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE +;; FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +;; DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +;; SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +;; CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +;; OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +;; +%define SHA224 +%include "avx512/mb_mgr_hmac_sha_256_submit_avx512.asm" diff --git a/src/spdk/intel-ipsec-mb/avx512/mb_mgr_hmac_sha_256_flush_avx512.asm b/src/spdk/intel-ipsec-mb/avx512/mb_mgr_hmac_sha_256_flush_avx512.asm new file mode 100644 index 000000000..023eb3454 --- /dev/null +++ b/src/spdk/intel-ipsec-mb/avx512/mb_mgr_hmac_sha_256_flush_avx512.asm @@ -0,0 +1,433 @@ +;; +;; Copyright (c) 2017-2018, Intel Corporation +;; +;; Redistribution and use in source and binary forms, with or without +;; modification, are permitted provided that the following conditions are met: +;; +;; * Redistributions of source code must retain the above copyright notice, +;; this list of conditions and the following disclaimer. +;; * Redistributions in binary form must reproduce the above copyright +;; notice, this list of conditions and the following disclaimer in the +;; documentation and/or other materials provided with the distribution. +;; * Neither the name of Intel Corporation nor the names of its contributors +;; may be used to endorse or promote products derived from this software +;; without specific prior written permission. +;; +;; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +;; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +;; IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +;; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE +;; FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +;; DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +;; SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +;; CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +;; OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +;; + +;; In System V AMD64 ABI +;; calle saves: RBX, RBP, R12-R15 +;; Windows x64 ABI +;; calle saves: RBX, RBP, RDI, RSI, RSP, R12-R15 +;; +;; Registers: RAX RBX RCX RDX RBP RSI RDI R8 R9 R10 R11 R12 R13 R14 R15 +;; ----------------------------------------------------------- +;; Windows clobbers: RAX RCX RDX R8 R9 R10 R11 +;; Windows preserves: RBX RBP RSI RDI R12 R13 R14 R15 +;; ----------------------------------------------------------- +;; Linux clobbers: RAX RCX RDX RSI RDI R8 R9 R10 R11 +;; Linux preserves: RBX RBP R12 R13 R14 R15 +;; ----------------------------------------------------------- +;; Clobbers ZMM0-31 + +%include "include/os.asm" +%include "job_aes_hmac.asm" +%include "mb_mgr_datastruct.asm" +%include "include/reg_sizes.asm" + +;; %define DO_DBGPRINT +%include "include/dbgprint.asm" + +extern sha256_x16_avx512 + +section .data +default rel +align 16 +byteswap: + dq 0x0405060700010203, 0x0c0d0e0f08090a0b + +align 32 +len_masks: + dq 0x000000000000FFFF, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000 + dq 0x00000000FFFF0000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000 + dq 0x0000FFFF00000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000 + dq 0xFFFF000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000 + dq 0x0000000000000000, 0x000000000000FFFF, 0x0000000000000000, 0x0000000000000000 + dq 0x0000000000000000, 0x00000000FFFF0000, 0x0000000000000000, 0x0000000000000000 + dq 0x0000000000000000, 0x0000FFFF00000000, 0x0000000000000000, 0x0000000000000000 + dq 0x0000000000000000, 0xFFFF000000000000, 0x0000000000000000, 0x0000000000000000 + dq 0x0000000000000000, 0x0000000000000000, 0x000000000000FFFF, 0x0000000000000000 + dq 0x0000000000000000, 0x0000000000000000, 0x00000000FFFF0000, 0x0000000000000000 + dq 0x0000000000000000, 0x0000000000000000, 0x0000FFFF00000000, 0x0000000000000000 + dq 0x0000000000000000, 0x0000000000000000, 0xFFFF000000000000, 0x0000000000000000 + dq 0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x000000000000FFFF + dq 0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x00000000FFFF0000 + dq 0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000FFFF00000000 + dq 0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0xFFFF000000000000 + +lane_1: dq 1 +lane_2: dq 2 +lane_3: dq 3 +lane_4: dq 4 +lane_5: dq 5 +lane_6: dq 6 +lane_7: dq 7 +lane_8: dq 8 +lane_9: dq 9 +lane_10: dq 10 +lane_11: dq 11 +lane_12: dq 12 +lane_13: dq 13 +lane_14: dq 14 +lane_15: dq 15 + +section .text + +%ifdef LINUX +%define arg1 rdi +%define arg2 rsi +%define arg3 rdx +%else +%define arg1 rcx +%define arg2 rdx +%define arg3 rsi +%endif + +%define state arg1 +%define job arg2 +%define len2 arg2 + + +; idx needs to be in rbp, r15 +%define idx rbp + +%define unused_lanes r10 +%define tmp5 r10 + +%define lane_data rbx +%define tmp2 rbx + +%define job_rax rax +%define tmp1 rax +%define size_offset rax +%define start_offset rax + +%define tmp3 arg1 + +%define extra_blocks arg2 +%define p arg2 + +%define tmp4 arg3 +%define tmp r9 + +%define len_upper r13 +%define idx_upper r14 + + +; we clobber rsi, rbp; called routine also clobbers rax, r9 to r15 +struc STACK +_gpr_save: resq 8 +_rsp_save: resq 1 +endstruc + +%define APPEND(a,b) a %+ b + +; JOB* flush_job_hmac_sha_224_avx512(MB_MGR_HMAC_SHA_256_OOO *state) +; JOB* flush_job_hmac_sha_256_avx512(MB_MGR_HMAC_SHA_256_OOO *state) +; arg 1 : state +align 32 +%ifdef SHA224 +MKGLOBAL(flush_job_hmac_sha_224_avx512,function,internal) +flush_job_hmac_sha_224_avx512: +%else +MKGLOBAL(flush_job_hmac_sha_256_avx512,function,internal) +flush_job_hmac_sha_256_avx512: +%endif + mov rax, rsp + sub rsp, STACK_size + and rsp, -32 + mov [rsp + _gpr_save + 8*0], rbx + mov [rsp + _gpr_save + 8*1], rbp + mov [rsp + _gpr_save + 8*2], r12 + mov [rsp + _gpr_save + 8*3], r13 + mov [rsp + _gpr_save + 8*4], r14 + mov [rsp + _gpr_save + 8*5], r15 +%ifndef LINUX + mov [rsp + _gpr_save + 8*6], rsi + mov [rsp + _gpr_save + 8*7], rdi +%endif + mov [rsp + _rsp_save], rax ; original SP + + ; if bit (32+3) is set, then all lanes are empty + cmp dword [state + _num_lanes_inuse_sha256], 0 + jz return_null + + ; find a lane with a non-null job + xor idx, idx + +%assign I 1 +%rep 15 + cmp qword [state + _ldata_sha256 + (I * _HMAC_SHA1_LANE_DATA_size) + _job_in_lane], 0 + cmovne idx, [rel APPEND(lane_,I)] +%assign I (I+1) +%endrep + +copy_lane_data: + ; copy idx to empty lanes + vmovdqa ymm0, [state + _lens_sha256] + mov tmp, [state + _args_data_ptr_sha256 + PTR_SZ*idx] + +%assign I 0 +%rep 16 + cmp qword [state + _ldata_sha256 + I * _HMAC_SHA1_LANE_DATA_size + _job_in_lane], 0 + jne APPEND(skip_,I) + mov [state + _args_data_ptr_sha256 + PTR_SZ*I], tmp + vpor ymm0, ymm0, [rel len_masks + 32*I] +APPEND(skip_,I): +%assign I (I+1) +%endrep + + vmovdqa [state + _lens_sha256 ], ymm0 + + vphminposuw xmm1, xmm0 + vpextrw DWORD(len2), xmm1, 0 ; min value + vpextrw DWORD(idx), xmm1, 1 ; min index (0...7) + + vmovdqa xmm2, [state + _lens_sha256 + 8*2] + vphminposuw xmm3, xmm2 + vpextrw DWORD(len_upper), xmm3, 0 ; min value + vpextrw DWORD(idx_upper), xmm3, 1 ; min index (8...F) + + cmp len2, len_upper + jle use_min + + vmovdqa xmm1, xmm3 + mov len2, len_upper + mov idx, idx_upper ; idx would be in range 0..7 + add idx, 8 ; to reflect that index is in 8..F range + +use_min: + cmp len2, 0 + je len_is_0 + + vpbroadcastw xmm1, xmm1 ; duplicate words across all lanes + vpsubw xmm0, xmm0, xmm1 + vmovdqa [state + _lens_sha256], xmm0 + vpsubw xmm2, xmm2, xmm1 + vmovdqa [state + _lens_sha256 + 8*2], xmm2 + + ; "state" and "args" are the same address, arg1 + ; len is arg2 + call sha256_x16_avx512 + ; state and idx are intact + +len_is_0: + ; process completed job "idx" + imul lane_data, idx, _HMAC_SHA1_LANE_DATA_size + lea lane_data, [state + _ldata_sha256 + lane_data] + mov DWORD(extra_blocks), [lane_data + _extra_blocks] + cmp extra_blocks, 0 + jne proc_extra_blocks + cmp dword [lane_data + _outer_done], 0 + jne end_loop + +proc_outer: + mov dword [lane_data + _outer_done], 1 + mov DWORD(size_offset), [lane_data + _size_offset] + mov qword [lane_data + _extra_block + size_offset], 0 + mov word [state + _lens_sha256 + 2*idx], 1 + lea tmp, [lane_data + _outer_block] + mov [state + _args_data_ptr_sha256 + PTR_SZ*idx], tmp + + vmovd xmm0, [state + _args_digest_sha256 + 4*idx + 0*SHA256_DIGEST_ROW_SIZE] + vpinsrd xmm0, xmm0, [state + _args_digest_sha256 + 4*idx + 1*SHA256_DIGEST_ROW_SIZE], 1 + vpinsrd xmm0, xmm0, [state + _args_digest_sha256 + 4*idx + 2*SHA256_DIGEST_ROW_SIZE], 2 + vpinsrd xmm0, xmm0, [state + _args_digest_sha256 + 4*idx + 3*SHA256_DIGEST_ROW_SIZE], 3 + vpshufb xmm0, xmm0, [rel byteswap] + vmovd xmm1, [state + _args_digest_sha256 + 4*idx + 4*SHA256_DIGEST_ROW_SIZE] + vpinsrd xmm1, xmm1, [state + _args_digest_sha256 + 4*idx + 5*SHA256_DIGEST_ROW_SIZE], 1 + vpinsrd xmm1, xmm1, [state + _args_digest_sha256 + 4*idx + 6*SHA256_DIGEST_ROW_SIZE], 2 +%ifndef SHA224 + vpinsrd xmm1, xmm1, [state + _args_digest_sha256 + 4*idx + 7*SHA256_DIGEST_ROW_SIZE], 3 +%endif + vpshufb xmm1, xmm1, [rel byteswap] + + vmovdqa [lane_data + _outer_block], xmm0 + vmovdqa [lane_data + _outer_block + 4*4], xmm1 +%ifdef SHA224 + mov dword [lane_data + _outer_block + 7*4], 0x80 +%endif + + mov job, [lane_data + _job_in_lane] + mov tmp, [job + _auth_key_xor_opad] + vmovdqu xmm0, [tmp] + vmovdqu xmm1, [tmp + 4*4] + vmovd [state + _args_digest_sha256 + 4*idx + 0*SHA256_DIGEST_ROW_SIZE], xmm0 + vpextrd [state + _args_digest_sha256 + 4*idx + 1*SHA256_DIGEST_ROW_SIZE], xmm0, 1 + vpextrd [state + _args_digest_sha256 + 4*idx + 2*SHA256_DIGEST_ROW_SIZE], xmm0, 2 + vpextrd [state + _args_digest_sha256 + 4*idx + 3*SHA256_DIGEST_ROW_SIZE], xmm0, 3 + vmovd [state + _args_digest_sha256 + 4*idx + 4*SHA256_DIGEST_ROW_SIZE], xmm1 + vpextrd [state + _args_digest_sha256 + 4*idx + 5*SHA256_DIGEST_ROW_SIZE], xmm1, 1 + vpextrd [state + _args_digest_sha256 + 4*idx + 6*SHA256_DIGEST_ROW_SIZE], xmm1, 2 + vpextrd [state + _args_digest_sha256 + 4*idx + 7*SHA256_DIGEST_ROW_SIZE], xmm1, 3 + jmp copy_lane_data + + align 16 +proc_extra_blocks: + mov DWORD(start_offset), [lane_data + _start_offset] + mov [state + _lens_sha256 + 2*idx], WORD(extra_blocks) + lea tmp, [lane_data + _extra_block + start_offset] + mov [state + _args_data_ptr_sha256 + PTR_SZ*idx], tmp + mov dword [lane_data + _extra_blocks], 0 + jmp copy_lane_data + +return_null: + xor job_rax, job_rax + jmp return + + align 16 +end_loop: + mov job_rax, [lane_data + _job_in_lane] + mov qword [lane_data + _job_in_lane], 0 + or dword [job_rax + _status], STS_COMPLETED_HMAC + mov unused_lanes, [state + _unused_lanes_sha256] + shl unused_lanes, 4 + or unused_lanes, idx + mov [state + _unused_lanes_sha256], unused_lanes + + sub dword [state + _num_lanes_inuse_sha256], 1 + + mov p, [job_rax + _auth_tag_output] + +%ifdef SHA224 + cmp qword [job_rax + _auth_tag_output_len_in_bytes], 14 + jne copy_full_digest +%else + cmp qword [job_rax + _auth_tag_output_len_in_bytes], 16 + jne copy_full_digest +%endif + + ;; copy SHA224 14 bytes / SHA256 16 bytes + mov DWORD(tmp), [state + _args_digest_sha256 + 4*idx + 0*SHA256_DIGEST_ROW_SIZE] + mov DWORD(tmp2), [state + _args_digest_sha256 + 4*idx + 1*SHA256_DIGEST_ROW_SIZE] + mov DWORD(tmp4), [state + _args_digest_sha256 + 4*idx + 2*SHA256_DIGEST_ROW_SIZE] + mov DWORD(tmp5), [state + _args_digest_sha256 + 4*idx + 3*SHA256_DIGEST_ROW_SIZE] + bswap DWORD(tmp) + bswap DWORD(tmp2) + bswap DWORD(tmp4) + bswap DWORD(tmp5) + mov [p + 0*4], DWORD(tmp) + mov [p + 1*4], DWORD(tmp2) + mov [p + 2*4], DWORD(tmp4) +%ifdef SHA224 + mov [p + 3*4], WORD(tmp5) +%else + mov [p + 3*4], DWORD(tmp5) +%endif + jmp clear_ret + +copy_full_digest: + ;; copy SHA224 28 bytes / SHA256 32 bytes + mov DWORD(tmp), [state + _args_digest_sha256 + 4*idx + 0*SHA256_DIGEST_ROW_SIZE] + mov DWORD(tmp2), [state + _args_digest_sha256 + 4*idx + 1*SHA256_DIGEST_ROW_SIZE] + mov DWORD(tmp4), [state + _args_digest_sha256 + 4*idx + 2*SHA256_DIGEST_ROW_SIZE] + mov DWORD(tmp5), [state + _args_digest_sha256 + 4*idx + 3*SHA256_DIGEST_ROW_SIZE] + bswap DWORD(tmp) + bswap DWORD(tmp2) + bswap DWORD(tmp4) + bswap DWORD(tmp5) + mov [p + 0*4], DWORD(tmp) + mov [p + 1*4], DWORD(tmp2) + mov [p + 2*4], DWORD(tmp4) + mov [p + 3*4], DWORD(tmp5) + + mov DWORD(tmp), [state + _args_digest_sha256 + 4*idx + 4*SHA256_DIGEST_ROW_SIZE] + mov DWORD(tmp2), [state + _args_digest_sha256 + 4*idx + 5*SHA256_DIGEST_ROW_SIZE] + mov DWORD(tmp4), [state + _args_digest_sha256 + 4*idx + 6*SHA256_DIGEST_ROW_SIZE] +%ifndef SHA224 + mov DWORD(tmp5), [state + _args_digest_sha256 + 4*idx + 7*SHA256_DIGEST_ROW_SIZE] +%endif + bswap DWORD(tmp) + bswap DWORD(tmp2) + bswap DWORD(tmp4) +%ifndef SHA224 + bswap DWORD(tmp5) +%endif + mov [p + 4*4], DWORD(tmp) + mov [p + 5*4], DWORD(tmp2) + mov [p + 6*4], DWORD(tmp4) +%ifndef SHA224 + mov [p + 7*4], DWORD(tmp5) +%endif + +clear_ret: + +%ifdef SAFE_DATA + vpxorq zmm0, zmm0 + + ;; Clear digest (28B/32B), outer_block (28B/32B) and extra_block (64B) + ;; of returned job and NULL jobs +%assign I 0 +%rep 16 + cmp qword [state + _ldata_sha256 + (I*_HMAC_SHA1_LANE_DATA_size) + _job_in_lane], 0 + jne APPEND(skip_clear_,I) + + ;; Clear digest (28 bytes for SHA-224, 32 bytes for SHA-256 bytes) +%assign J 0 +%rep 7 + mov dword [state + _args_digest_sha256 + SHA256_DIGEST_WORD_SIZE*I + J*SHA256_DIGEST_ROW_SIZE], 0 +%assign J (J+1) +%endrep +%ifndef SHA224 + mov dword [state + _args_digest_sha256 + SHA256_DIGEST_WORD_SIZE*I + 7*SHA256_DIGEST_ROW_SIZE], 0 +%endif + + lea lane_data, [state + _ldata_sha256 + (I*_HMAC_SHA1_LANE_DATA_size)] + ;; Clear first 64 bytes of extra_block + vmovdqu64 [lane_data + _extra_block], zmm0 + + ;; Clear first 28 bytes (SHA-224) or 32 bytes (SHA-256) of outer_block +%ifdef SHA224 + vmovdqa64 [lane_data + _outer_block], xmm0 + mov qword [lane_data + _outer_block + 16], 0 + mov dword [lane_data + _outer_block + 24], 0 +%else + vmovdqu64 [lane_data + _outer_block], ymm0 +%endif + +APPEND(skip_clear_,I): +%assign I (I+1) +%endrep + +%endif ;; SAFE_DATA + +return: + vzeroupper + + mov rbx, [rsp + _gpr_save + 8*0] + mov rbp, [rsp + _gpr_save + 8*1] + mov r12, [rsp + _gpr_save + 8*2] + mov r13, [rsp + _gpr_save + 8*3] + mov r14, [rsp + _gpr_save + 8*4] + mov r15, [rsp + _gpr_save + 8*5] +%ifndef LINUX + mov rsi, [rsp + _gpr_save + 8*6] + mov rdi, [rsp + _gpr_save + 8*7] +%endif + mov rsp, [rsp + _rsp_save] ; original SP + + ret + +%ifdef LINUX +section .note.GNU-stack noalloc noexec nowrite progbits +%endif diff --git a/src/spdk/intel-ipsec-mb/avx512/mb_mgr_hmac_sha_256_submit_avx512.asm b/src/spdk/intel-ipsec-mb/avx512/mb_mgr_hmac_sha_256_submit_avx512.asm new file mode 100644 index 000000000..baadef492 --- /dev/null +++ b/src/spdk/intel-ipsec-mb/avx512/mb_mgr_hmac_sha_256_submit_avx512.asm @@ -0,0 +1,445 @@ +;; +;; Copyright (c) 2017-2018, Intel Corporation +;; +;; Redistribution and use in source and binary forms, with or without +;; modification, are permitted provided that the following conditions are met: +;; +;; * Redistributions of source code must retain the above copyright notice, +;; this list of conditions and the following disclaimer. +;; * Redistributions in binary form must reproduce the above copyright +;; notice, this list of conditions and the following disclaimer in the +;; documentation and/or other materials provided with the distribution. +;; * Neither the name of Intel Corporation nor the names of its contributors +;; may be used to endorse or promote products derived from this software +;; without specific prior written permission. +;; +;; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +;; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +;; IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +;; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE +;; FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +;; DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +;; SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +;; CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +;; OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +;; + +;; In System V AMD64 ABI +;; calle saves: RBX, RBP, R12-R15 +;; Windows x64 ABI +;; calle saves: RBX, RBP, RDI, RSI, RSP, R12-R15 +;; +;; Registers: RAX RBX RCX RDX RBP RSI RDI R8 R9 R10 R11 R12 R13 R14 R15 +;; ----------------------------------------------------------- +;; Windows clobbers: RAX RCX RDX R8 R9 R10 R11 +;; Windows preserves: RBX RBP RSI RDI R12 R13 R14 R15 +;; ----------------------------------------------------------- +;; Linux clobbers: RAX RCX RDX RSI RDI R8 R9 R10 R11 +;; Linux preserves: RBX RBP R12 R13 R14 R15 +;; ----------------------------------------------------------- +;; Clobbers ZMM0-31 + +%include "include/os.asm" +%include "job_aes_hmac.asm" +%include "mb_mgr_datastruct.asm" +%include "include/reg_sizes.asm" +%include "include/memcpy.asm" + +;; %define DO_DBGPRINT +%include "include/dbgprint.asm" + +extern sha256_x16_avx512 + +section .data +default rel + +align 16 +byteswap: + dq 0x0405060700010203, 0x0c0d0e0f08090a0b + +section .text + +%ifdef LINUX +%define arg1 rdi +%define arg2 rsi +%define arg3 rcx +%define arg4 rdx +%else +%define arg1 rcx +%define arg2 rdx +%define arg3 rdi +%define arg4 rsi +%endif + +%define state arg1 +%define job arg2 +%define len2 arg2 + +; idx needs to be in rbp, r15 +%define last_len rbp +%define idx rbp + +%define p r11 +%define start_offset r11 + +%define unused_lanes rbx +%define p2 rbx +%define tmp4 rbx + +%define job_rax rax +%define len rax + +%define size_offset arg3 +%define tmp2 arg3 + +%define lane arg4 +%define tmp3 arg4 + +%define extra_blocks r8 +%define tmp r9 +%define lane_data r10 + +%define len_upper r13 +%define idx_upper r14 + +; we clobber rbx, rsi, rdi, rbp; called routine also clobbers r9 to r15 +struc STACK +_gpr_save: resq 8 +_rsp_save: resq 1 +endstruc + +; JOB* FUNC(MB_MGR_HMAC_SHA_256_OOO *state, JOB_AES_HMAC *job) +; arg 1 : rcx : state +; arg 2 : rdx : job + align 32 +%ifdef SHA224 +MKGLOBAL(submit_job_hmac_sha_224_avx512,function,internal) +submit_job_hmac_sha_224_avx512: +%else +MKGLOBAL(submit_job_hmac_sha_256_avx512,function,internal) +submit_job_hmac_sha_256_avx512: +%endif + mov rax, rsp + sub rsp, STACK_size + and rsp, -32 + mov [rsp + _gpr_save + 8*0], rbx + mov [rsp + _gpr_save + 8*1], rbp + mov [rsp + _gpr_save + 8*2], r12 + mov [rsp + _gpr_save + 8*3], r13 + mov [rsp + _gpr_save + 8*4], r14 + mov [rsp + _gpr_save + 8*5], r15 +%ifndef LINUX + mov [rsp + _gpr_save + 8*6], rsi + mov [rsp + _gpr_save + 8*7], rdi +%endif + mov [rsp + _rsp_save], rax ; original SP + + mov unused_lanes, [state + _unused_lanes_sha256] + mov lane, unused_lanes + and lane, 0xF ;; just a nibble + shr unused_lanes, 4 + + imul lane_data, lane, _HMAC_SHA1_LANE_DATA_size + lea lane_data, [state + _ldata_sha256 + lane_data] + mov [state + _unused_lanes_sha256], unused_lanes + + add dword [state + _num_lanes_inuse_sha256], 1 + + mov len, [job + _msg_len_to_hash_in_bytes] + mov tmp, len + shr tmp, 6 ; divide by 64, len in terms of blocks + + mov [lane_data + _job_in_lane], job + mov dword [lane_data + _outer_done], 0 + mov [state + _lens_sha256 + 2*lane], WORD(tmp) + + mov last_len, len + and last_len, 63 + lea extra_blocks, [last_len + 9 + 63] + shr extra_blocks, 6 + mov [lane_data + _extra_blocks], DWORD(extra_blocks) + + mov p, [job + _src] + add p, [job + _hash_start_src_offset_in_bytes] + mov [state + _args_data_ptr_sha256 + PTR_SZ*lane], p + + cmp len, 64 + jb copy_lt64 + +fast_copy: + vmovdqu32 zmm0, [p - 64 + len] + vmovdqu32 [lane_data + _extra_block], zmm0 +end_fast_copy: + + mov size_offset, extra_blocks + shl size_offset, 6 + sub size_offset, last_len + add size_offset, 64-8 + mov [lane_data + _size_offset], DWORD(size_offset) + mov start_offset, 64 + sub start_offset, last_len + mov [lane_data + _start_offset], DWORD(start_offset) + + lea tmp, [8*64 + 8*len] + bswap tmp + mov [lane_data + _extra_block + size_offset], tmp + + mov tmp, [job + _auth_key_xor_ipad] + vmovdqu xmm0, [tmp] + vmovdqu xmm1, [tmp + 4*4] + vmovd [state + _args_digest_sha256 + 4*lane + 0*SHA256_DIGEST_ROW_SIZE], xmm0 + vpextrd [state + _args_digest_sha256 + 4*lane + 1*SHA256_DIGEST_ROW_SIZE], xmm0, 1 + vpextrd [state + _args_digest_sha256 + 4*lane + 2*SHA256_DIGEST_ROW_SIZE], xmm0, 2 + vpextrd [state + _args_digest_sha256 + 4*lane + 3*SHA256_DIGEST_ROW_SIZE], xmm0, 3 + vmovd [state + _args_digest_sha256 + 4*lane + 4*SHA256_DIGEST_ROW_SIZE], xmm1 + vpextrd [state + _args_digest_sha256 + 4*lane + 5*SHA256_DIGEST_ROW_SIZE], xmm1, 1 + vpextrd [state + _args_digest_sha256 + 4*lane + 6*SHA256_DIGEST_ROW_SIZE], xmm1, 2 + vpextrd [state + _args_digest_sha256 + 4*lane + 7*SHA256_DIGEST_ROW_SIZE], xmm1, 3 + + test len, ~63 + jnz ge64_bytes + +lt64_bytes: + mov [state + _lens_sha256 + 2*lane], WORD(extra_blocks) + lea tmp, [lane_data + _extra_block + start_offset] + mov [state + _args_data_ptr_sha256 + PTR_SZ*lane], tmp + mov dword [lane_data + _extra_blocks], 0 + +ge64_bytes: + cmp dword [state + _num_lanes_inuse_sha256], 0x10 ; all 16 lanes used? + jne return_null + jmp start_loop + + align 16 +start_loop: + ; Find min length + vmovdqa xmm0, [state + _lens_sha256] + vphminposuw xmm1, xmm0 + vpextrw DWORD(len2), xmm1, 0 ; min value + vpextrw DWORD(idx), xmm1, 1 ; min index (0...7) + + vmovdqa xmm2, [state + _lens_sha256 + 8*2] + vphminposuw xmm3, xmm2 + vpextrw DWORD(len_upper), xmm3, 0 ; min value + vpextrw DWORD(idx_upper), xmm3, 1 ; min index (8...F) + + cmp len2, len_upper + jle use_min + + vmovdqa xmm1, xmm3 + mov len2, len_upper + mov idx, idx_upper ; idx is in range 0..7 + add idx, 8 ; to reflect that real index is in 8..F range +use_min: + cmp len2, 0 + je len_is_0 + + vpbroadcastw xmm1, xmm1 ; duplicate words across all lanes + vpsubw xmm0, xmm0, xmm1 + vmovdqa [state + _lens_sha256 + 0*2], xmm0 + vpsubw xmm2, xmm2, xmm1 + vmovdqa [state + _lens_sha256 + 8*2], xmm2 + + ; "state" and "args" are the same address, arg1 + ; len is arg2 + call sha256_x16_avx512 + ; state and idx are intact + +len_is_0: + ; process completed job "idx" + imul lane_data, idx, _HMAC_SHA1_LANE_DATA_size + lea lane_data, [state + _ldata_sha256 + lane_data] + mov DWORD(extra_blocks), [lane_data + _extra_blocks] + cmp extra_blocks, 0 + jne proc_extra_blocks + cmp dword [lane_data + _outer_done], 0 + jne end_loop + +proc_outer: + mov dword [lane_data + _outer_done], 1 + mov DWORD(size_offset), [lane_data + _size_offset] + mov qword [lane_data + _extra_block + size_offset], 0 + mov word [state + _lens_sha256 + 2*idx], 1 + lea tmp, [lane_data + _outer_block] + mov job, [lane_data + _job_in_lane] + mov [state + _args_data_ptr_sha256 + PTR_SZ*idx], tmp + + vmovd xmm0, [state + _args_digest_sha256 + 4*idx + 0*SHA256_DIGEST_ROW_SIZE] + vpinsrd xmm0, xmm0, [state + _args_digest_sha256 + 4*idx + 1*SHA256_DIGEST_ROW_SIZE], 1 + vpinsrd xmm0, xmm0, [state + _args_digest_sha256 + 4*idx + 2*SHA256_DIGEST_ROW_SIZE], 2 + vpinsrd xmm0, xmm0, [state + _args_digest_sha256 + 4*idx + 3*SHA256_DIGEST_ROW_SIZE], 3 + vpshufb xmm0, xmm0, [rel byteswap] + vmovd xmm1, [state + _args_digest_sha256 + 4*idx + 4*SHA256_DIGEST_ROW_SIZE] + vpinsrd xmm1, xmm1, [state + _args_digest_sha256 + 4*idx + 5*SHA256_DIGEST_ROW_SIZE], 1 + vpinsrd xmm1, xmm1, [state + _args_digest_sha256 + 4*idx + 6*SHA256_DIGEST_ROW_SIZE], 2 +%ifndef SHA224 + vpinsrd xmm1, xmm1, [state + _args_digest_sha256 + 4*idx + 7*SHA256_DIGEST_ROW_SIZE], 3 +%endif + vpshufb xmm1, xmm1, [rel byteswap] + vmovdqa [lane_data + _outer_block], xmm0 + vmovdqa [lane_data + _outer_block + 4*4], xmm1 +%ifdef SHA224 + mov dword [lane_data + _outer_block + 7*4], 0x80 +%endif + + mov tmp, [job + _auth_key_xor_opad] + vmovdqu xmm0, [tmp] + vmovdqu xmm1, [tmp + 4*4] + vmovd [state + _args_digest_sha256 + 4*idx + 0*SHA256_DIGEST_ROW_SIZE], xmm0 + vpextrd [state + _args_digest_sha256 + 4*idx + 1*SHA256_DIGEST_ROW_SIZE], xmm0, 1 + vpextrd [state + _args_digest_sha256 + 4*idx + 2*SHA256_DIGEST_ROW_SIZE], xmm0, 2 + vpextrd [state + _args_digest_sha256 + 4*idx + 3*SHA256_DIGEST_ROW_SIZE], xmm0, 3 + vmovd [state + _args_digest_sha256 + 4*idx + 4*SHA256_DIGEST_ROW_SIZE], xmm1 + vpextrd [state + _args_digest_sha256 + 4*idx + 5*SHA256_DIGEST_ROW_SIZE], xmm1, 1 + vpextrd [state + _args_digest_sha256 + 4*idx + 6*SHA256_DIGEST_ROW_SIZE], xmm1, 2 + vpextrd [state + _args_digest_sha256 + 4*idx + 7*SHA256_DIGEST_ROW_SIZE], xmm1, 3 + + jmp start_loop + + align 16 +proc_extra_blocks: + mov DWORD(start_offset), [lane_data + _start_offset] + mov [state + _lens_sha256 + 2*idx], WORD(extra_blocks) + lea tmp, [lane_data + _extra_block + start_offset] + mov [state + _args_data_ptr_sha256 + 8*idx], tmp + mov dword [lane_data + _extra_blocks], 0 + jmp start_loop + + align 16 +copy_lt64: + ;; less than one message block of data + ;; beginning of source block + ;; destination extrablock but backwards by len from where 0x80 pre-populated + lea p2, [lane_data + _extra_block + 64] + sub p2, len + memcpy_avx2_64_1 p2, p, len, tmp, tmp2, ymm0, ymm1 + mov unused_lanes, [state + _unused_lanes_sha256] + jmp end_fast_copy + +return_null: + xor job_rax, job_rax + jmp return + + align 16 +end_loop: + mov job_rax, [lane_data + _job_in_lane] + mov unused_lanes, [state + _unused_lanes_sha256] + mov qword [lane_data + _job_in_lane], 0 + or dword [job_rax + _status], STS_COMPLETED_HMAC + shl unused_lanes, 4 + or unused_lanes, idx + mov [state + _unused_lanes_sha256], unused_lanes + sub dword [state + _num_lanes_inuse_sha256], 1 + + vzeroupper + + mov p, [job_rax + _auth_tag_output] + +%ifdef SHA224 + cmp qword [job_rax + _auth_tag_output_len_in_bytes], 14 + jne copy_full_digest +%else + cmp qword [job_rax + _auth_tag_output_len_in_bytes], 16 + jne copy_full_digest +%endif + + ;; copy 14 bytes for SHA224 // 16 bytes for SHA256 + mov DWORD(tmp), [state + _args_digest_sha256 + 4*idx + 0*SHA256_DIGEST_ROW_SIZE] + mov DWORD(tmp2), [state + _args_digest_sha256 + 4*idx + 1*SHA256_DIGEST_ROW_SIZE] + mov DWORD(tmp3), [state + _args_digest_sha256 + 4*idx + 2*SHA256_DIGEST_ROW_SIZE] + mov DWORD(tmp4), [state + _args_digest_sha256 + 4*idx + 3*SHA256_DIGEST_ROW_SIZE] + bswap DWORD(tmp) + bswap DWORD(tmp2) + bswap DWORD(tmp3) + bswap DWORD(tmp4) + mov [p + 0*4], DWORD(tmp) + mov [p + 1*4], DWORD(tmp2) + mov [p + 2*4], DWORD(tmp3) +%ifdef SHA224 + mov [p + 3*4], WORD(tmp4) +%else + mov [p + 3*4], DWORD(tmp4) +%endif + jmp clear_ret +copy_full_digest: + ;; copy 28 bytes for SHA224 // 32 bytes for SHA256 + mov DWORD(tmp), [state + _args_digest_sha256 + 4*idx + 0*SHA256_DIGEST_ROW_SIZE] + mov DWORD(tmp2), [state + _args_digest_sha256 + 4*idx + 1*SHA256_DIGEST_ROW_SIZE] + mov DWORD(tmp3), [state + _args_digest_sha256 + 4*idx + 2*SHA256_DIGEST_ROW_SIZE] + mov DWORD(tmp4), [state + _args_digest_sha256 + 4*idx + 3*SHA256_DIGEST_ROW_SIZE] + bswap DWORD(tmp) + bswap DWORD(tmp2) + bswap DWORD(tmp3) + bswap DWORD(tmp4) + mov [p + 0*4], DWORD(tmp) + mov [p + 1*4], DWORD(tmp2) + mov [p + 2*4], DWORD(tmp3) + mov [p + 3*4], DWORD(tmp4) + + mov DWORD(tmp), [state + _args_digest_sha256 + 4*idx + 4*SHA256_DIGEST_ROW_SIZE] + mov DWORD(tmp2), [state + _args_digest_sha256 + 4*idx + 5*SHA256_DIGEST_ROW_SIZE] + mov DWORD(tmp3), [state + _args_digest_sha256 + 4*idx + 6*SHA256_DIGEST_ROW_SIZE] +%ifndef SHA224 + mov DWORD(tmp4), [state + _args_digest_sha256 + 4*idx + 7*SHA256_DIGEST_ROW_SIZE] +%endif + bswap DWORD(tmp) + bswap DWORD(tmp2) + bswap DWORD(tmp3) +%ifndef SHA224 + bswap DWORD(tmp4) +%endif + mov [p + 4*4], DWORD(tmp) + mov [p + 5*4], DWORD(tmp2) + mov [p + 6*4], DWORD(tmp3) +%ifndef SHA224 + mov [p + 7*4], DWORD(tmp4) +%endif + +clear_ret: + +%ifdef SAFE_DATA + ;; Clear digest (28B/32B), outer_block (28B/32B) and extra_block (64B) of returned job +%assign J 0 +%rep 7 + mov dword [state + _args_digest_sha256 + SHA256_DIGEST_WORD_SIZE*idx + J*SHA256_DIGEST_ROW_SIZE], 0 +%assign J (J+1) +%endrep +%ifndef SHA224 + mov dword [state + _args_digest_sha256 + SHA256_DIGEST_WORD_SIZE*idx + 7*SHA256_DIGEST_ROW_SIZE], 0 +%endif + + vpxorq zmm0, zmm0 + imul lane_data, idx, _HMAC_SHA1_LANE_DATA_size + lea lane_data, [state + _ldata_sha256 + lane_data] + ;; Clear first 64 bytes of extra_block + vmovdqu64 [lane_data + _extra_block], zmm0 + + ;; Clear first 28 bytes (SHA-224) or 32 bytes (SHA-256) of outer_block +%ifdef SHA224 + vmovdqa64 [lane_data + _outer_block], xmm0 + mov qword [lane_data + _outer_block + 16], 0 + mov dword [lane_data + _outer_block + 24], 0 +%else + vmovdqu64 [lane_data + _outer_block], ymm0 +%endif +%endif ;; SAFE_DATA + +return: + mov rbx, [rsp + _gpr_save + 8*0] + mov rbp, [rsp + _gpr_save + 8*1] + mov r12, [rsp + _gpr_save + 8*2] + mov r13, [rsp + _gpr_save + 8*3] + mov r14, [rsp + _gpr_save + 8*4] + mov r15, [rsp + _gpr_save + 8*5] +%ifndef LINUX + mov rsi, [rsp + _gpr_save + 8*6] + mov rdi, [rsp + _gpr_save + 8*7] +%endif + mov rsp, [rsp + _rsp_save] ; original SP + + ret + +%ifdef LINUX +section .note.GNU-stack noalloc noexec nowrite progbits +%endif diff --git a/src/spdk/intel-ipsec-mb/avx512/mb_mgr_hmac_sha_384_flush_avx512.asm b/src/spdk/intel-ipsec-mb/avx512/mb_mgr_hmac_sha_384_flush_avx512.asm new file mode 100644 index 000000000..698052730 --- /dev/null +++ b/src/spdk/intel-ipsec-mb/avx512/mb_mgr_hmac_sha_384_flush_avx512.asm @@ -0,0 +1,29 @@ +;; +;; Copyright (c) 2017-2018, Intel Corporation +;; +;; Redistribution and use in source and binary forms, with or without +;; modification, are permitted provided that the following conditions are met: +;; +;; * Redistributions of source code must retain the above copyright notice, +;; this list of conditions and the following disclaimer. +;; * Redistributions in binary form must reproduce the above copyright +;; notice, this list of conditions and the following disclaimer in the +;; documentation and/or other materials provided with the distribution. +;; * Neither the name of Intel Corporation nor the names of its contributors +;; may be used to endorse or promote products derived from this software +;; without specific prior written permission. +;; +;; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +;; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +;; IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +;; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE +;; FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +;; DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +;; SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +;; CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +;; OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +;; + +%define SHA384 +%include "avx512/mb_mgr_hmac_sha_512_flush_avx512.asm" diff --git a/src/spdk/intel-ipsec-mb/avx512/mb_mgr_hmac_sha_384_submit_avx512.asm b/src/spdk/intel-ipsec-mb/avx512/mb_mgr_hmac_sha_384_submit_avx512.asm new file mode 100644 index 000000000..0e9f611de --- /dev/null +++ b/src/spdk/intel-ipsec-mb/avx512/mb_mgr_hmac_sha_384_submit_avx512.asm @@ -0,0 +1,29 @@ +;; +;; Copyright (c) 2017-2018, Intel Corporation +;; +;; Redistribution and use in source and binary forms, with or without +;; modification, are permitted provided that the following conditions are met: +;; +;; * Redistributions of source code must retain the above copyright notice, +;; this list of conditions and the following disclaimer. +;; * Redistributions in binary form must reproduce the above copyright +;; notice, this list of conditions and the following disclaimer in the +;; documentation and/or other materials provided with the distribution. +;; * Neither the name of Intel Corporation nor the names of its contributors +;; may be used to endorse or promote products derived from this software +;; without specific prior written permission. +;; +;; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +;; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +;; IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +;; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE +;; FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +;; DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +;; SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +;; CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +;; OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +;; + +%define SHA384 +%include "avx512/mb_mgr_hmac_sha_512_submit_avx512.asm" diff --git a/src/spdk/intel-ipsec-mb/avx512/mb_mgr_hmac_sha_512_flush_avx512.asm b/src/spdk/intel-ipsec-mb/avx512/mb_mgr_hmac_sha_512_flush_avx512.asm new file mode 100644 index 000000000..7d7e56b40 --- /dev/null +++ b/src/spdk/intel-ipsec-mb/avx512/mb_mgr_hmac_sha_512_flush_avx512.asm @@ -0,0 +1,384 @@ +;; +;; Copyright (c) 2017-2018, Intel Corporation +;; +;; Redistribution and use in source and binary forms, with or without +;; modification, are permitted provided that the following conditions are met: +;; +;; * Redistributions of source code must retain the above copyright notice, +;; this list of conditions and the following disclaimer. +;; * Redistributions in binary form must reproduce the above copyright +;; notice, this list of conditions and the following disclaimer in the +;; documentation and/or other materials provided with the distribution. +;; * Neither the name of Intel Corporation nor the names of its contributors +;; may be used to endorse or promote products derived from this software +;; without specific prior written permission. +;; +;; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +;; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +;; IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +;; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE +;; FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +;; DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +;; SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +;; CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +;; OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +;; + +;; In System V AMD64 ABI +;; calle saves: RBX, RBP, R12-R15 +;; Windows x64 ABI +;; calle saves: RBX, RBP, RDI, RSI, R12-R15 +;; +;; Clobbers ZMM0-31 + +%include "include/os.asm" +%include "job_aes_hmac.asm" +%include "mb_mgr_datastruct.asm" +%include "include/reg_sizes.asm" + +extern sha512_x8_avx512 + +section .data +default rel + +align 16 +dupw: ;ddq 0x01000100010001000100010001000100 + dq 0x0100010001000100, 0x0100010001000100 + +align 16 +byteswap: ;ddq 0x08090a0b0c0d0e0f0001020304050607 + dq 0x0001020304050607, 0x08090a0b0c0d0e0f + +align 16 +len_masks: + ;ddq 0x0000000000000000000000000000FFFF + dq 0x000000000000FFFF, 0x0000000000000000 + ;ddq 0x000000000000000000000000FFFF0000 + dq 0x00000000FFFF0000, 0x0000000000000000 + ;ddq 0x00000000000000000000FFFF00000000 + dq 0x0000FFFF00000000, 0x0000000000000000 + ;ddq 0x0000000000000000FFFF000000000000 + dq 0xFFFF000000000000, 0x0000000000000000 + ;ddq 0x000000000000FFFF0000000000000000 + dq 0x0000000000000000, 0x000000000000FFFF + ;ddq 0x00000000FFFF00000000000000000000 + dq 0x0000000000000000, 0x00000000FFFF0000 + ;ddq 0x0000FFFF000000000000000000000000 + dq 0x0000000000000000, 0x0000FFFF00000000 + ;ddq 0xFFFF0000000000000000000000000000 + dq 0x0000000000000000, 0xFFFF000000000000 + +lane_1: dq 1 +lane_2: dq 2 +lane_3: dq 3 +lane_4: dq 4 +lane_5: dq 5 +lane_6: dq 6 +lane_7: dq 7 + +section .text + +%ifdef LINUX +%define arg1 rdi +%define arg2 rsi +%else +%define arg1 rcx +%define arg2 rdx +%endif + +%define state arg1 +%define job arg2 +%define len2 arg2 + + +; idx needs to be in rbp +%define idx rbp + +%define unused_lanes rbx +%define lane_data rbx +%define tmp2 rbx + +%define job_rax rax + +%define size_offset rax +%define tmp rax +%define start_offset rax + +%define extra_blocks arg2 +%define p arg2 + +%define tmp4 r8 +%define tmp5 r9 +%define tmp6 r10 + +struc STACK +_gpr_save: resq 7 ; rbx, rbp, r12-r15, rdi (windows) +_rsp_save: resq 1 +endstruc + +%define APPEND(a,b) a %+ b + +%ifndef SHA384 +; JOB* flush_job_hmac_sha_384_avx512(MB_MGR_HMAC_SHA_512_OOO *state) +; arg 1 : state +%define SHA_X_DIGEST_SIZE 512 +MKGLOBAL(flush_job_hmac_sha_512_avx512,function,internal) +align 64 +flush_job_hmac_sha_512_avx512: +%else +; JOB* flush_job_hmac_sha_512_avx512(MB_MGR_HMAC_SHA_512_OOO *state) +; arg 1 : state +%define SHA_X_DIGEST_SIZE 384 +MKGLOBAL(flush_job_hmac_sha_384_avx512,function,internal) +align 64 +flush_job_hmac_sha_384_avx512: +%endif + mov rax, rsp + sub rsp, STACK_size + and rsp, -32 + mov [rsp + _gpr_save + 8*0], rbx + mov [rsp + _gpr_save + 8*1], rbp + mov [rsp + _gpr_save + 8*2], r12 + mov [rsp + _gpr_save + 8*3], r13 + mov [rsp + _gpr_save + 8*4], r14 + mov [rsp + _gpr_save + 8*5], r15 +%ifndef LINUX + mov [rsp + _gpr_save + 8*6], rdi +%endif + mov [rsp + _rsp_save], rax ; original SP + + mov unused_lanes, [state + _unused_lanes_sha512] + bt unused_lanes, 32+3 + jc return_null + + ; find a lane with a non-null job + xor idx, idx +%assign I 1 +%rep 7 + cmp qword [state + _ldata_sha512 + I * _SHA512_LANE_DATA_size + _job_in_lane_sha512], 0 + cmovne idx, [rel APPEND(lane_, I)] +%assign I (I+1) +%endrep + +copy_lane_data: + ; copy good lane (idx) to empty lanes + vmovdqa xmm0, [state + _lens_sha512] + mov tmp, [state + _args_sha512 + _data_ptr_sha512 + PTR_SZ*idx] + +%assign I 0 +%rep 8 + cmp qword [state + _ldata_sha512 + I * _SHA512_LANE_DATA_size + _job_in_lane_sha512], 0 + jne APPEND(skip_,I) + mov [state + _args_sha512 + _data_ptr_sha512 + PTR_SZ*I], tmp + vpor xmm0, xmm0, [rel len_masks + 16*I] +APPEND(skip_,I): +%assign I (I+1) +%endrep + + vmovdqa [state + _lens_sha512], xmm0 + + vphminposuw xmm1, xmm0 + vpextrw DWORD(len2), xmm1, 0 ; min value + vpextrw DWORD(idx), xmm1, 1 ; min index (0...7) + cmp len2, 0 + je len_is_0 + + vpshufb xmm1, [rel dupw] ; duplicate words across all 8 lanes + vpsubw xmm0, xmm0, xmm1 + vmovdqa [state + _lens_sha512], xmm0 + + ; "state" and "args" are the same address, arg1 + ; len is arg2 + call sha512_x8_avx512 + ; state and idx are intact + +len_is_0: + ; process completed job "idx" + imul lane_data, idx, _SHA512_LANE_DATA_size + lea lane_data, [state + _ldata_sha512 + lane_data] + mov DWORD(extra_blocks), [lane_data + _extra_blocks_sha512] + cmp extra_blocks, 0 + jne proc_extra_blocks + cmp dword [lane_data + _outer_done_sha512], 0 + jne end_loop + +proc_outer: + mov dword [lane_data + _outer_done_sha512], 1 + mov DWORD(size_offset), [lane_data + _size_offset_sha512] + mov qword [lane_data + _extra_block_sha512 + size_offset], 0 + mov word [state + _lens_sha512 + 2*idx], 1 + lea tmp, [lane_data + _outer_block_sha512] + mov job, [lane_data + _job_in_lane_sha512] + mov [state + _args_data_ptr_sha512 + PTR_SZ*idx], tmp + + ; move digest into data location + %assign I 0 + %rep (SHA_X_DIGEST_SIZE / (8*16)) + vmovq xmm0, [state + _args_digest_sha512 + SHA512_DIGEST_WORD_SIZE*idx + 2*I*SHA512_DIGEST_ROW_SIZE] + vpinsrq xmm0, [state + _args_digest_sha512 + SHA512_DIGEST_WORD_SIZE*idx + (2*I + 1)*SHA512_DIGEST_ROW_SIZE], 1 + vpshufb xmm0, [rel byteswap] + vmovdqa [lane_data + _outer_block_sha512 + I*2*SHA512_DIGEST_WORD_SIZE], xmm0 + %assign I (I+1) + %endrep + + ; move the opad key into digest + mov tmp, [job + _auth_key_xor_opad] + + %assign I 0 + %rep 4 + vmovdqu xmm0, [tmp + I * 16] + vmovq [state + _args_digest_sha512 + SHA512_DIGEST_WORD_SIZE*idx + (2*I + 0)*SHA512_DIGEST_ROW_SIZE], xmm0 + vpextrq [state + _args_digest_sha512 + SHA512_DIGEST_WORD_SIZE*idx + (2*I + 1)*SHA512_DIGEST_ROW_SIZE], xmm0, 1 + %assign I (I+1) + %endrep + + jmp copy_lane_data + + align 32 +proc_extra_blocks: + mov DWORD(start_offset), [lane_data + _start_offset_sha512] + mov [state + _lens_sha512 + 2*idx], WORD(extra_blocks) + lea tmp, [lane_data + _extra_block_sha512 + start_offset] + mov [state + _args_data_ptr_sha512 + PTR_SZ*idx], tmp + mov dword [lane_data + _extra_blocks_sha512], 0 + jmp copy_lane_data + +return_null: + xor job_rax, job_rax + jmp return + + align 32 +end_loop: + mov job_rax, [lane_data + _job_in_lane_sha512] + mov qword [lane_data + _job_in_lane_sha512], 0 + or dword [job_rax + _status], STS_COMPLETED_HMAC + mov unused_lanes, [state + _unused_lanes_sha512] + shl unused_lanes, 4 + or unused_lanes, idx + mov [state + _unused_lanes_sha512], unused_lanes + + mov p, [job_rax + _auth_tag_output] + +%if (SHA_X_DIGEST_SIZE != 384) + cmp qword [job_rax + _auth_tag_output_len_in_bytes], 32 + jne copy_full_digest +%else + cmp qword [job_rax + _auth_tag_output_len_in_bytes], 24 + jne copy_full_digest +%endif + ;; copy 32 bytes for SHA512 / 24 bytes for SHA384 + mov QWORD(tmp2), [state + _args_digest_sha512 + SHA512_DIGEST_WORD_SIZE*idx + 0*SHA512_DIGEST_ROW_SIZE] + mov QWORD(tmp4), [state + _args_digest_sha512 + SHA512_DIGEST_WORD_SIZE*idx + 1*SHA512_DIGEST_ROW_SIZE] + mov QWORD(tmp6), [state + _args_digest_sha512 + SHA512_DIGEST_WORD_SIZE*idx + 2*SHA512_DIGEST_ROW_SIZE] +%if (SHA_X_DIGEST_SIZE != 384) + mov QWORD(tmp5), [state + _args_digest_sha512 + SHA512_DIGEST_WORD_SIZE*idx + 3*SHA512_DIGEST_ROW_SIZE] +%endif + bswap QWORD(tmp2) + bswap QWORD(tmp4) + bswap QWORD(tmp6) +%if (SHA_X_DIGEST_SIZE != 384) + bswap QWORD(tmp5) +%endif + mov [p + 0*8], QWORD(tmp2) + mov [p + 1*8], QWORD(tmp4) + mov [p + 2*8], QWORD(tmp6) +%if (SHA_X_DIGEST_SIZE != 384) + mov [p + 3*8], QWORD(tmp5) +%endif + jmp clear_ret + +copy_full_digest: + ;; copy 64 bytes for SHA512 / 48 bytes for SHA384 + mov QWORD(tmp2), [state + _args_digest_sha512 + SHA512_DIGEST_WORD_SIZE*idx + 0*SHA512_DIGEST_ROW_SIZE] + mov QWORD(tmp4), [state + _args_digest_sha512 + SHA512_DIGEST_WORD_SIZE*idx + 1*SHA512_DIGEST_ROW_SIZE] + mov QWORD(tmp6), [state + _args_digest_sha512 + SHA512_DIGEST_WORD_SIZE*idx + 2*SHA512_DIGEST_ROW_SIZE] + mov QWORD(tmp5), [state + _args_digest_sha512 + SHA512_DIGEST_WORD_SIZE*idx + 3*SHA512_DIGEST_ROW_SIZE] + bswap QWORD(tmp2) + bswap QWORD(tmp4) + bswap QWORD(tmp6) + bswap QWORD(tmp5) + mov [p + 0*8], QWORD(tmp2) + mov [p + 1*8], QWORD(tmp4) + mov [p + 2*8], QWORD(tmp6) + mov [p + 3*8], QWORD(tmp5) + + mov QWORD(tmp2), [state + _args_digest_sha512 + SHA512_DIGEST_WORD_SIZE*idx + 4*SHA512_DIGEST_ROW_SIZE] + mov QWORD(tmp4), [state + _args_digest_sha512 + SHA512_DIGEST_WORD_SIZE*idx + 5*SHA512_DIGEST_ROW_SIZE] +%if (SHA_X_DIGEST_SIZE != 384) + mov QWORD(tmp6), [state + _args_digest_sha512 + SHA512_DIGEST_WORD_SIZE*idx + 6*SHA512_DIGEST_ROW_SIZE] + mov QWORD(tmp5), [state + _args_digest_sha512 + SHA512_DIGEST_WORD_SIZE*idx + 7*SHA512_DIGEST_ROW_SIZE] +%endif + bswap QWORD(tmp2) + bswap QWORD(tmp4) +%if (SHA_X_DIGEST_SIZE != 384) + bswap QWORD(tmp6) + bswap QWORD(tmp5) +%endif + mov [p + 4*8], QWORD(tmp2) + mov [p + 5*8], QWORD(tmp4) +%if (SHA_X_DIGEST_SIZE != 384) + mov [p + 6*8], QWORD(tmp6) + mov [p + 7*8], QWORD(tmp5) +%endif + +clear_ret: + +%ifdef SAFE_DATA + vpxorq zmm0, zmm0 + + ;; Clear digest (48B/64B), outer_block (48B/64B) and extra_block (128B) of returned job +%assign I 0 +%rep 8 + cmp qword [state + _ldata_sha512 + (I*_SHA512_LANE_DATA_size) + _job_in_lane_sha512], 0 + jne APPEND(skip_clear_,I) + + ;; Clear digest (48 bytes for SHA-384, 64 bytes for SHA-512 bytes) +%assign J 0 +%rep 6 + mov qword [state + _args_digest_sha512 + SHA512_DIGEST_WORD_SIZE*I + J*SHA512_DIGEST_ROW_SIZE], 0 +%assign J (J+1) +%endrep +%if (SHA_X_DIGEST_SIZE != 384) + mov qword [state + _args_digest_sha512 + SHA512_DIGEST_WORD_SIZE*I + 6*SHA512_DIGEST_ROW_SIZE], 0 + mov qword [state + _args_digest_sha512 + SHA512_DIGEST_WORD_SIZE*I + 7*SHA512_DIGEST_ROW_SIZE], 0 +%endif + + lea lane_data, [state + _ldata_sha512 + (I*_SHA512_LANE_DATA_size)] + ;; Clear first 128 bytes of extra_block + vmovdqu64 [lane_data + _extra_block], zmm0 + vmovdqu64 [lane_data + _extra_block + 64], zmm0 + + ;; Clear first 48 bytes (SHA-384) or 64 bytes (SHA-512) of outer_block +%if (SHA_X_DIGEST_SIZE == 384) + vmovdqu64 [lane_data + _outer_block], ymm0 + vmovdqa64 [lane_data + _outer_block + 32], xmm0 +%else + vmovdqu64 [lane_data + _outer_block], zmm0 +%endif + +APPEND(skip_clear_,I): +%assign I (I+1) +%endrep + +%endif ;; SAFE_DATA + +return: + vzeroupper + + mov rbx, [rsp + _gpr_save + 8*0] + mov rbp, [rsp + _gpr_save + 8*1] + mov r12, [rsp + _gpr_save + 8*2] + mov r13, [rsp + _gpr_save + 8*3] + mov r14, [rsp + _gpr_save + 8*4] + mov r15, [rsp + _gpr_save + 8*5] +%ifndef LINUX + mov rdi, [rsp + _gpr_save + 8*6] +%endif + mov rsp, [rsp + _rsp_save] ; original SP + + ret + +%ifdef LINUX +section .note.GNU-stack noalloc noexec nowrite progbits +%endif diff --git a/src/spdk/intel-ipsec-mb/avx512/mb_mgr_hmac_sha_512_submit_avx512.asm b/src/spdk/intel-ipsec-mb/avx512/mb_mgr_hmac_sha_512_submit_avx512.asm new file mode 100644 index 000000000..a2b66e54f --- /dev/null +++ b/src/spdk/intel-ipsec-mb/avx512/mb_mgr_hmac_sha_512_submit_avx512.asm @@ -0,0 +1,413 @@ +;; +;; Copyright (c) 2017-2018, Intel Corporation +;; +;; Redistribution and use in source and binary forms, with or without +;; modification, are permitted provided that the following conditions are met: +;; +;; * Redistributions of source code must retain the above copyright notice, +;; this list of conditions and the following disclaimer. +;; * Redistributions in binary form must reproduce the above copyright +;; notice, this list of conditions and the following disclaimer in the +;; documentation and/or other materials provided with the distribution. +;; * Neither the name of Intel Corporation nor the names of its contributors +;; may be used to endorse or promote products derived from this software +;; without specific prior written permission. +;; +;; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +;; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +;; IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +;; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE +;; FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +;; DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +;; SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +;; CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +;; OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +;; + +;; In System V AMD64 ABI +;; calle saves: RBX, RBP, R12-R15 +;; Windows x64 ABI +;; calle saves: RBX, RBP, RDI, RSI, RSP, R12-R15 +;; +;; Clobbers ZMM0-31 + +%include "include/os.asm" +%include "job_aes_hmac.asm" +%include "mb_mgr_datastruct.asm" +%include "include/reg_sizes.asm" +%include "include/memcpy.asm" + +extern sha512_x8_avx512 + +section .data +default rel + +align 16 +dupw: ;ddq 0x01000100010001000100010001000100 + dq 0x0100010001000100, 0x0100010001000100 + +align 16 +byteswap: ;ddq 0x08090a0b0c0d0e0f0001020304050607 + dq 0x0001020304050607, 0x08090a0b0c0d0e0f + +section .text + +%ifdef LINUX +%define arg1 rdi +%define arg2 rsi +%define arg3 rcx +%define arg4 rdx +%else +%define arg1 rcx +%define arg2 rdx +%define arg3 rdi +%define arg4 rsi +%endif + +%define state arg1 +%define job arg2 +%define len2 arg2 + + +; idx needs to be in rbp, r13, r14, r16 +%define last_len rbp +%define idx rbp + +%define p r11 +%define start_offset r11 + +%define unused_lanes rbx +%define tmp4 rbx + +%define job_rax rax +%define len rax + +%define size_offset arg3 +%define tmp2 arg3 + +%define lane arg4 +%define tmp3 arg4 + +%define extra_blocks r8 + +%define tmp r9 +%define p2 r9 + +%define lane_data r10 + +; Define stack usage + +; we clobber rbx, rsi, rdi, rbp; called routine also clobbers r12 +struc STACK +_gpr_save: resq 8 +_rsp_save: resq 1 +endstruc + +; JOB* FUNC(MB_MGR_HMAC_sha_512_OOO *state, JOB_AES_HMAC *job) +; arg 1 : rcx : state +; arg 2 : rdx : job +align 64 +%ifndef SHA384 +MKGLOBAL(submit_job_hmac_sha_512_avx512,function,internal) +%define SHA_X_DIGEST_SIZE 512 +submit_job_hmac_sha_512_avx512: +%else +MKGLOBAL(submit_job_hmac_sha_384_avx512,function,internal) +%define SHA_X_DIGEST_SIZE 384 +submit_job_hmac_sha_384_avx512: +%endif + + mov rax, rsp + sub rsp, STACK_size + and rsp, -32 + mov [rsp + _gpr_save + 8*0], rbx + mov [rsp + _gpr_save + 8*1], rbp + mov [rsp + _gpr_save + 8*2], r12 + mov [rsp + _gpr_save + 8*3], r13 + mov [rsp + _gpr_save + 8*4], r14 + mov [rsp + _gpr_save + 8*5], r15 +%ifndef LINUX + mov [rsp + _gpr_save + 8*6], rsi + mov [rsp + _gpr_save + 8*7], rdi +%endif + mov [rsp + _rsp_save], rax ; original SP + + mov unused_lanes, [state + _unused_lanes_sha512] + mov lane, unused_lanes + and lane, 15 + shr unused_lanes, 4 + imul lane_data, lane, _SHA512_LANE_DATA_size + lea lane_data, [state + _ldata_sha512 + lane_data] + mov [state + _unused_lanes_sha512], unused_lanes + mov len, [job + _msg_len_to_hash_in_bytes] + mov tmp, len + shr tmp, 7 ; divide by 128, len in terms of blocks + + mov [lane_data + _job_in_lane_sha512], job + mov dword [lane_data + _outer_done_sha512], 0 + mov [state + _lens_sha512 + 2*lane], WORD(tmp) ; 2 is word size in bytes + + mov last_len, len + and last_len, 127 + lea extra_blocks, [last_len + 17 + 127] + shr extra_blocks, 7 + mov [lane_data + _extra_blocks_sha512], DWORD(extra_blocks) + + mov p, [job + _src] + add p, [job + _hash_start_src_offset_in_bytes] + mov [state + _args_data_ptr_sha512 + PTR_SZ*lane], p + + cmp len, 128 + jb copy_lt128 + +fast_copy: + add p, len + vmovdqu32 zmm0, [p - 128 + 0*64] + vmovdqu32 zmm1, [p - 128 + 1*64] + vmovdqu32 [lane_data + _extra_block_sha512 + 0*64], zmm0 + vmovdqu32 [lane_data + _extra_block_sha512 + 1*64], zmm1 +end_fast_copy: + + mov size_offset, extra_blocks + shl size_offset, 7 + sub size_offset, last_len + add size_offset, 128-8 + mov [lane_data + _size_offset_sha512], DWORD(size_offset) + mov start_offset, 128 + sub start_offset, last_len + mov [lane_data + _start_offset_sha512], DWORD(start_offset) + + lea tmp, [8*128 + 8*len] + bswap tmp + mov [lane_data + _extra_block_sha512 + size_offset], tmp + + mov tmp, [job + _auth_key_xor_ipad] + +%assign I 0 +%rep 4 + vmovdqu xmm0, [tmp + I * 2 * SHA512_DIGEST_WORD_SIZE] + vmovq [state + _args_digest_sha512 + SHA512_DIGEST_WORD_SIZE*lane + (2*I + 0)*SHA512_DIGEST_ROW_SIZE], xmm0 + vpextrq [state + _args_digest_sha512 + SHA512_DIGEST_WORD_SIZE*lane + (2*I + 1)*SHA512_DIGEST_ROW_SIZE], xmm0, 1 +%assign I (I+1) +%endrep + + test len, ~127 + jnz ge128_bytes + +lt128_bytes: + mov [state + _lens_sha512 + 2*lane], WORD(extra_blocks) + lea tmp, [lane_data + _extra_block_sha512 + start_offset] + mov [state + _args_data_ptr_sha512 + PTR_SZ*lane], tmp ;; 8 to hold a UINT8 + mov dword [lane_data + _extra_blocks_sha512], 0 + +ge128_bytes: + cmp unused_lanes, 0xf + jne return_null + jmp start_loop + + align 32 +start_loop: + ; Find min length + vmovdqa xmm0, [state + _lens_sha512] + vphminposuw xmm1, xmm0 + vpextrw DWORD(len2), xmm1, 0 ; min value + vpextrw DWORD(idx), xmm1, 1 ; min index (0...7) + cmp len2, 0 + je len_is_0 + + vpshufb xmm1, [rel dupw] ; duplicate words across all 8 lanes + vpsubw xmm0, xmm0, xmm1 + vmovdqa [state + _lens_sha512], xmm0 + + ; "state" and "args" are the same address, arg1 + ; len is arg2 + call sha512_x8_avx512 + ; state and idx are intact + +len_is_0: + ; process completed job "idx" + imul lane_data, idx, _SHA512_LANE_DATA_size + lea lane_data, [state + _ldata_sha512 + lane_data] + mov DWORD(extra_blocks), [lane_data + _extra_blocks_sha512] + cmp extra_blocks, 0 + jne proc_extra_blocks + cmp dword [lane_data + _outer_done_sha512], 0 + jne end_loop + +proc_outer: + mov dword [lane_data + _outer_done_sha512], 1 + mov DWORD(size_offset), [lane_data + _size_offset_sha512] + mov qword [lane_data + _extra_block_sha512 + size_offset], 0 + mov word [state + _lens_sha512 + 2*idx], 1 + lea tmp, [lane_data + _outer_block_sha512] + mov job, [lane_data + _job_in_lane_sha512] + mov [state + _args_data_ptr_sha512 + PTR_SZ*idx], tmp + +%assign I 0 +%rep (SHA_X_DIGEST_SIZE / (8 * 16)) + vmovq xmm0, [state + _args_digest_sha512 + SHA512_DIGEST_WORD_SIZE*idx + (2*I + 0)*SHA512_DIGEST_ROW_SIZE] + vpinsrq xmm0, [state + _args_digest_sha512 + SHA512_DIGEST_WORD_SIZE*idx + (2*I + 1)*SHA512_DIGEST_ROW_SIZE], 1 + vpshufb xmm0, [rel byteswap] + vmovdqa [lane_data + _outer_block_sha512 + I * 2 * SHA512_DIGEST_WORD_SIZE], xmm0 +%assign I (I+1) +%endrep + + mov tmp, [job + _auth_key_xor_opad] +%assign I 0 +%rep 4 + vmovdqu xmm0, [tmp + I * 16] + vmovq [state + _args_digest_sha512 + SHA512_DIGEST_WORD_SIZE*idx + (2*I+0)*SHA512_DIGEST_ROW_SIZE], xmm0 + vpextrq [state + _args_digest_sha512 + SHA512_DIGEST_WORD_SIZE*idx + (2*I + 1)*SHA512_DIGEST_ROW_SIZE], xmm0, 1 +%assign I (I+1) +%endrep + + jmp start_loop + + align 32 +proc_extra_blocks: + mov DWORD(start_offset), [lane_data + _start_offset_sha512] + mov [state + _lens_sha512 + 2*idx], WORD(extra_blocks) + lea tmp, [lane_data + _extra_block_sha512 + start_offset] + mov [state + _args_data_ptr_sha512 + PTR_SZ*idx], tmp ;; idx is index of shortest length message + mov dword [lane_data + _extra_blocks_sha512], 0 + jmp start_loop + + align 32 +copy_lt128: + ;; less than one message block of data + ;; destination extra block but backwards by len from where 0x80 pre-populated + lea p2, [lane_data + _extra_block + 128] + sub p2, len + memcpy_avx2_128_1 p2, p, len, tmp4, tmp2, ymm0, ymm1, ymm2, ymm3 + mov unused_lanes, [state + _unused_lanes_sha512] + jmp end_fast_copy + +return_null: + xor job_rax, job_rax + jmp return + + align 32 +end_loop: + mov job_rax, [lane_data + _job_in_lane_sha512] + mov unused_lanes, [state + _unused_lanes_sha512] + mov qword [lane_data + _job_in_lane_sha512], 0 + or dword [job_rax + _status], STS_COMPLETED_HMAC + shl unused_lanes, 4 + or unused_lanes, idx + mov [state + _unused_lanes_sha512], unused_lanes + + mov p, [job_rax + _auth_tag_output] + + vzeroupper + +%if (SHA_X_DIGEST_SIZE != 384) + cmp qword [job_rax + _auth_tag_output_len_in_bytes], 32 + jne copy_full_digest +%else + cmp qword [job_rax + _auth_tag_output_len_in_bytes], 24 + jne copy_full_digest +%endif + + ;; copy 32 bytes for SHA512 / 24 bytes for SHA384 + mov QWORD(tmp), [state + _args_digest_sha512 + SHA512_DIGEST_WORD_SIZE*idx + 0*SHA512_DIGEST_ROW_SIZE] + mov QWORD(tmp2), [state + _args_digest_sha512 + SHA512_DIGEST_WORD_SIZE*idx + 1*SHA512_DIGEST_ROW_SIZE] + mov QWORD(tmp3), [state + _args_digest_sha512 + SHA512_DIGEST_WORD_SIZE*idx + 2*SHA512_DIGEST_ROW_SIZE] +%if (SHA_X_DIGEST_SIZE != 384) + mov QWORD(tmp4), [state + _args_digest_sha512 + SHA512_DIGEST_WORD_SIZE*idx + 3*SHA512_DIGEST_ROW_SIZE] +%endif + bswap QWORD(tmp) + bswap QWORD(tmp2) + bswap QWORD(tmp3) +%if (SHA_X_DIGEST_SIZE != 384) + bswap QWORD(tmp4) +%endif + mov [p + 0*8], QWORD(tmp) + mov [p + 1*8], QWORD(tmp2) + mov [p + 2*8], QWORD(tmp3) +%if (SHA_X_DIGEST_SIZE != 384) + mov [p + 3*8], QWORD(tmp4) +%endif + jmp clear_ret + +copy_full_digest: + ;; copy 64 bytes for SHA512 / 48 bytes for SHA384 + mov QWORD(tmp), [state + _args_digest_sha512 + SHA512_DIGEST_WORD_SIZE*idx + 0*SHA512_DIGEST_ROW_SIZE] + mov QWORD(tmp2), [state + _args_digest_sha512 + SHA512_DIGEST_WORD_SIZE*idx + 1*SHA512_DIGEST_ROW_SIZE] + mov QWORD(tmp3), [state + _args_digest_sha512 + SHA512_DIGEST_WORD_SIZE*idx + 2*SHA512_DIGEST_ROW_SIZE] + mov QWORD(tmp4), [state + _args_digest_sha512 + SHA512_DIGEST_WORD_SIZE*idx + 3*SHA512_DIGEST_ROW_SIZE] + bswap QWORD(tmp) + bswap QWORD(tmp2) + bswap QWORD(tmp3) + bswap QWORD(tmp4) + mov [p + 0*8], QWORD(tmp) + mov [p + 1*8], QWORD(tmp2) + mov [p + 2*8], QWORD(tmp3) + mov [p + 3*8], QWORD(tmp4) + + mov QWORD(tmp), [state + _args_digest_sha512 + SHA512_DIGEST_WORD_SIZE*idx + 4*SHA512_DIGEST_ROW_SIZE] + mov QWORD(tmp2), [state + _args_digest_sha512 + SHA512_DIGEST_WORD_SIZE*idx + 5*SHA512_DIGEST_ROW_SIZE] +%if (SHA_X_DIGEST_SIZE != 384) + mov QWORD(tmp3), [state + _args_digest_sha512 + SHA512_DIGEST_WORD_SIZE*idx + 6*SHA512_DIGEST_ROW_SIZE] + mov QWORD(tmp4), [state + _args_digest_sha512 + SHA512_DIGEST_WORD_SIZE*idx + 7*SHA512_DIGEST_ROW_SIZE] +%endif + bswap QWORD(tmp) + bswap QWORD(tmp2) +%if (SHA_X_DIGEST_SIZE != 384) + bswap QWORD(tmp3) + bswap QWORD(tmp4) +%endif + mov [p + 4*8], QWORD(tmp) + mov [p + 5*8], QWORD(tmp2) +%if (SHA_X_DIGEST_SIZE != 384) + mov [p + 6*8], QWORD(tmp3) + mov [p + 7*8], QWORD(tmp4) +%endif + +clear_ret: + +%ifdef SAFE_DATA + ;; Clear digest (48B/64B), outer_block (48B/64B) and extra_block (128B) of returned job +%assign J 0 +%rep 6 + mov qword [state + _args_digest_sha512 + SHA512_DIGEST_WORD_SIZE*idx + J*SHA512_DIGEST_ROW_SIZE], 0 +%assign J (J+1) +%endrep +%if (SHA_X_DIGEST_SIZE != 384) + mov qword [state + _args_digest_sha512 + SHA512_DIGEST_WORD_SIZE*idx + 6*SHA256_DIGEST_ROW_SIZE], 0 + mov qword [state + _args_digest_sha512 + SHA512_DIGEST_WORD_SIZE*idx + 7*SHA256_DIGEST_ROW_SIZE], 0 +%endif + + vpxorq zmm0, zmm0 + imul lane_data, idx, _SHA512_LANE_DATA_size + lea lane_data, [state + _ldata_sha512 + lane_data] + ;; Clear first 128 bytes of extra_block + vmovdqu64 [lane_data + _extra_block], zmm0 + vmovdqu64 [lane_data + _extra_block + 64], zmm0 + + ;; Clear first 48 bytes (SHA-384) or 64 bytes (SHA-512) of outer_block +%if (SHA_X_DIGEST_SIZE == 384) + vmovdqu64 [lane_data + _outer_block], ymm0 + vmovdqa64 [lane_data + _outer_block + 32], xmm0 +%else + vmovdqu64 [lane_data + _outer_block], zmm0 +%endif +%endif ;; SAFE_DATA + +return: + mov rbx, [rsp + _gpr_save + 8*0] + mov rbp, [rsp + _gpr_save + 8*1] + mov r12, [rsp + _gpr_save + 8*2] + mov r13, [rsp + _gpr_save + 8*3] + mov r14, [rsp + _gpr_save + 8*4] + mov r15, [rsp + _gpr_save + 8*5] +%ifndef LINUX + mov rsi, [rsp + _gpr_save + 8*6] + mov rdi, [rsp + _gpr_save + 8*7] +%endif + mov rsp, [rsp + _rsp_save] ; original SP + ret + +%ifdef LINUX +section .note.GNU-stack noalloc noexec nowrite progbits +%endif diff --git a/src/spdk/intel-ipsec-mb/avx512/mb_mgr_hmac_submit_avx512.asm b/src/spdk/intel-ipsec-mb/avx512/mb_mgr_hmac_submit_avx512.asm new file mode 100644 index 000000000..2fe8482a9 --- /dev/null +++ b/src/spdk/intel-ipsec-mb/avx512/mb_mgr_hmac_submit_avx512.asm @@ -0,0 +1,402 @@ +;; +;; Copyright (c) 2012-2018, Intel Corporation +;; +;; Redistribution and use in source and binary forms, with or without +;; modification, are permitted provided that the following conditions are met: +;; +;; * Redistributions of source code must retain the above copyright notice, +;; this list of conditions and the following disclaimer. +;; * Redistributions in binary form must reproduce the above copyright +;; notice, this list of conditions and the following disclaimer in the +;; documentation and/or other materials provided with the distribution. +;; * Neither the name of Intel Corporation nor the names of its contributors +;; may be used to endorse or promote products derived from this software +;; without specific prior written permission. +;; +;; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +;; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +;; IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +;; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE +;; FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +;; DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +;; SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +;; CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +;; OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +;; + +;; In System V AMD64 ABI +;; calle saves: RBX, RBP, R12-R15 +;; Windows x64 ABI +;; calle saves: RBX, RBP, RDI, RSI, RSP, R12-R15 +;; +;; Registers: RAX RBX RCX RDX RBP RSI RDI R8 R9 R10 R11 R12 R13 R14 R15 +;; ----------------------------------------------------------- +;; Windows clobbers: RAX RCX RDX R8 R9 R10 R11 +;; Windows preserves: RBX RBP RSI RDI R12 R13 R14 R15 +;; ----------------------------------------------------------- +;; Linux clobbers: RAX RCX RDX RSI RDI R8 R9 R10 R11 +;; Linux preserves: RBX RBP R12 R13 R14 R15 +;; ----------------------------------------------------------- +;; Clobbers ZMM0-31 + +%include "include/os.asm" +%include "job_aes_hmac.asm" +%include "mb_mgr_datastruct.asm" +%include "include/reg_sizes.asm" +%include "include/memcpy.asm" + +;; %define DO_DBGPRINT +%include "include/dbgprint.asm" + +extern sha1_x16_avx512 + +section .data +default rel + +align 16 +byteswap: + dq 0x0405060700010203 + dq 0x0c0d0e0f08090a0b + +section .text + +%if 1 +%ifdef LINUX +%define arg1 rdi +%define arg2 rsi +%define reg3 rcx +%define reg4 rdx +%else +%define arg1 rcx +%define arg2 rdx +%define reg3 rdi +%define reg4 rsi +%endif + +%define state arg1 +%define job arg2 +%define len2 arg2 + + +; idx needs to be in rbx, rdi, rbp +%define last_len rbp +%define idx rbp + +%define p r11 +%define start_offset r11 + +%define unused_lanes r12 +%define tmp4 r12 + +%define job_rax rax +%define len rax + +%define size_offset reg3 +%define tmp2 reg3 + +%define lane reg4 +%define tmp3 reg4 + +%define extra_blocks r8 + +%define tmp r9 +%define p2 r9 + +%define lane_data r10 +%define num_lanes_inuse r12 +%define len_upper r13 +%define idx_upper r14 +%endif + +; we clobber rsi, rdi, rbp, r12; called routine clobbers also r9-r15 +struc STACK +_gpr_save: resq 7 +_rsp_save: resq 1 +endstruc + +; JOB* submit_job_hmac_avx(MB_MGR_HMAC_SHA_1_OOO *state, JOB_AES_HMAC *job) +; arg 1 : rcx : state +; arg 2 : rdx : job +MKGLOBAL(submit_job_hmac_avx512,function,internal) +submit_job_hmac_avx512: + + mov rax, rsp + sub rsp, STACK_size + and rsp, -32 ; align to 32 byte boundary + mov [rsp + _gpr_save + 8*0], rbp + mov [rsp + _gpr_save + 8*1], r12 + mov [rsp + _gpr_save + 8*2], r13 + mov [rsp + _gpr_save + 8*3], r14 + mov [rsp + _gpr_save + 8*4], r15 +%ifndef LINUX + mov [rsp + _gpr_save + 8*5], rsi + mov [rsp + _gpr_save + 8*6], rdi +%endif + mov [rsp + _rsp_save], rax + DBGPRINTL "---------- enter sha1 submit -----------" + + mov unused_lanes, [state + _unused_lanes] + mov lane, unused_lanes + and lane, 0xF ;; just a nibble + shr unused_lanes, 4 + imul lane_data, lane, _HMAC_SHA1_LANE_DATA_size + lea lane_data, [state + _ldata + lane_data] + mov [state + _unused_lanes], unused_lanes + DBGPRINTL64 "lane", lane + DBGPRINTL64 "unused_lanes", unused_lanes + + add dword [state + _num_lanes_inuse_sha1], 1 + + mov len, [job + _msg_len_to_hash_in_bytes] + mov tmp, len + shr tmp, 6 ; divide by 64, len in terms of blocks + + mov [lane_data + _job_in_lane], job + mov dword [lane_data + _outer_done], 0 + mov [state + _lens + 2*lane], WORD(tmp) + + mov last_len, len + DBGPRINTL64 "last_len", last_len + and last_len, 63 + lea extra_blocks, [last_len + 9 + 63] + shr extra_blocks, 6 + DBGPRINTL64 "extra_blocks", extra_blocks + mov [lane_data + _extra_blocks], DWORD(extra_blocks) + + mov p, [job + _src] + add p, [job + _hash_start_src_offset_in_bytes] + mov [state + _args_data_ptr + PTR_SZ*lane], p + cmp len, 64 + jb copy_lt64 + +fast_copy: + vmovdqu32 zmm0, [p - 64 + len] + vmovdqu32 [lane_data + _extra_block], zmm0 + +end_fast_copy: + mov size_offset, extra_blocks + shl size_offset, 6 + sub size_offset, last_len + add size_offset, 64-8 + mov [lane_data + _size_offset], DWORD(size_offset) + mov start_offset, 64 + sub start_offset, last_len + mov [lane_data + _start_offset], DWORD(start_offset) + + lea tmp, [8*64 + 8*len] + bswap tmp + mov [lane_data + _extra_block + size_offset], tmp + + mov tmp, [job + _auth_key_xor_ipad] + vmovdqu xmm0, [tmp] + mov DWORD(tmp), [tmp + 4*4] + vmovd [state + _args_digest + SHA1_DIGEST_WORD_SIZE*lane + 0*SHA1_DIGEST_ROW_SIZE], xmm0 + vpextrd [state + _args_digest + SHA1_DIGEST_WORD_SIZE*lane + 1*SHA1_DIGEST_ROW_SIZE], xmm0, 1 + vpextrd [state + _args_digest + SHA1_DIGEST_WORD_SIZE*lane + 2*SHA1_DIGEST_ROW_SIZE], xmm0, 2 + vpextrd [state + _args_digest + SHA1_DIGEST_WORD_SIZE*lane + 3*SHA1_DIGEST_ROW_SIZE], xmm0, 3 + mov [state + _args_digest + SHA1_DIGEST_WORD_SIZE*lane + 4*SHA1_DIGEST_ROW_SIZE], DWORD(tmp) + + test len, ~63 + jnz ge64_bytes + +lt64_bytes: + DBGPRINTL64 "lt64_bytes extra_blocks", extra_blocks + DBGPRINTL64 "lt64_bytes start_offset", start_offset + mov [state + _lens + 2*lane], WORD(extra_blocks) + lea tmp, [lane_data + _extra_block + start_offset] + mov [state + _args_data_ptr + PTR_SZ*lane], tmp + mov dword [lane_data + _extra_blocks], 0 + +ge64_bytes: + mov DWORD(num_lanes_inuse), [state + _num_lanes_inuse_sha1] + cmp num_lanes_inuse, 0x10 ; all 16 lanes used? + jne return_null + jmp start_loop + + align 16 +start_loop: + ; Find min length + vmovdqa xmm0, [state + _lens] + vphminposuw xmm1, xmm0 + vpextrw DWORD(len2), xmm1, 0 ; min value + vpextrw DWORD(idx), xmm1, 1 ; min index (0...7) + + vmovdqa xmm2, [state + _lens + 8*2] + vphminposuw xmm3, xmm2 + vpextrw DWORD(len_upper), xmm3, 0 ; min value + vpextrw DWORD(idx_upper), xmm3, 1 ; min index (8...F) + + cmp len2, len_upper + jle use_min + + vmovdqa xmm1, xmm3 + mov len2, len_upper + mov idx, idx_upper ; idx would be in range 0..7 + add idx, 8 ; to reflect that index is in 8..F range + +use_min: + cmp len2, 0 + je len_is_0 + + DBGPRINTL64 "min_length", len2 + DBGPRINTL64 "min_length index ", idx + + vpbroadcastw xmm1, xmm1 + DBGPRINTL_XMM "SUBMIT lens after shuffle", xmm1 + + vpsubw xmm0, xmm0, xmm1 + vmovdqa [state + _lens + 0*2], xmm0 + vpsubw xmm2, xmm2, xmm1 + vmovdqa [state + _lens + 8*2], xmm2 + DBGPRINTL_XMM "lengths after subtraction (0..7)", xmm0 + DBGPRINTL_XMM "lengths after subtraction (8..F)", xmm2 + + ; "state" and "args" are the same address, arg1 + ; len is arg2 + call sha1_x16_avx512 + ; state and idx are intact + +len_is_0: + ; process completed job "idx" + imul lane_data, idx, _HMAC_SHA1_LANE_DATA_size + lea lane_data, [state + _ldata + lane_data] + mov DWORD(extra_blocks), [lane_data + _extra_blocks] + cmp extra_blocks, 0 + jne proc_extra_blocks + cmp dword [lane_data + _outer_done], 0 + jne end_loop + +proc_outer: + mov dword [lane_data + _outer_done], 1 + mov DWORD(size_offset), [lane_data + _size_offset] + mov qword [lane_data + _extra_block + size_offset], 0 + mov word [state + _lens + 2*idx], 1 + lea tmp, [lane_data + _outer_block] + mov job, [lane_data + _job_in_lane] + mov [state + _args_data_ptr + PTR_SZ*idx], tmp + + vmovd xmm0, [state + _args_digest + SHA1_DIGEST_WORD_SIZE*idx + 0*SHA1_DIGEST_ROW_SIZE] + vpinsrd xmm0, xmm0, [state + _args_digest + SHA1_DIGEST_WORD_SIZE*idx + 1*SHA1_DIGEST_ROW_SIZE], 1 + vpinsrd xmm0, xmm0, [state + _args_digest + SHA1_DIGEST_WORD_SIZE*idx + 2*SHA1_DIGEST_ROW_SIZE], 2 + vpinsrd xmm0, xmm0, [state + _args_digest + SHA1_DIGEST_WORD_SIZE*idx + 3*SHA1_DIGEST_ROW_SIZE], 3 + vpshufb xmm0, xmm0, [rel byteswap] + mov DWORD(tmp), [state + _args_digest + SHA1_DIGEST_WORD_SIZE*idx + 4*SHA1_DIGEST_ROW_SIZE] + bswap DWORD(tmp) + vmovdqa [lane_data + _outer_block], xmm0 + mov [lane_data + _outer_block + 4*SHA1_DIGEST_WORD_SIZE], DWORD(tmp) + + mov tmp, [job + _auth_key_xor_opad] + vmovdqu xmm0, [tmp] + mov DWORD(tmp), [tmp + 4*4] + vmovd [state + _args_digest + SHA1_DIGEST_WORD_SIZE*idx + 0*SHA1_DIGEST_ROW_SIZE], xmm0 + vpextrd [state + _args_digest + SHA1_DIGEST_WORD_SIZE*idx + 1*SHA1_DIGEST_ROW_SIZE], xmm0, 1 + vpextrd [state + _args_digest + SHA1_DIGEST_WORD_SIZE*idx + 2*SHA1_DIGEST_ROW_SIZE], xmm0, 2 + vpextrd [state + _args_digest + SHA1_DIGEST_WORD_SIZE*idx + 3*SHA1_DIGEST_ROW_SIZE], xmm0, 3 + mov [state + _args_digest + SHA1_DIGEST_WORD_SIZE*idx + 4*SHA1_DIGEST_ROW_SIZE], DWORD(tmp) + + jmp start_loop + + align 16 +proc_extra_blocks: + mov DWORD(start_offset), [lane_data + _start_offset] + mov [state + _lens + 2*idx], WORD(extra_blocks) + lea tmp, [lane_data + _extra_block + start_offset] + mov [state + _args_data_ptr + PTR_SZ*idx], tmp + mov dword [lane_data + _extra_blocks], 0 + jmp start_loop + + align 16 +copy_lt64: + ;; less than one message block of data + ;; beginning of source block + ;; destination extrablock but backwards by len from where 0x80 pre-populated + lea p2, [lane_data + _extra_block + 64] + sub p2, len + memcpy_avx2_64_1 p2, p, len, tmp4, tmp2, ymm0, ymm1 + mov unused_lanes, [state + _unused_lanes] + jmp end_fast_copy + +return_null: + xor job_rax, job_rax + jmp return + + align 16 +end_loop: + mov job_rax, [lane_data + _job_in_lane] + or dword [job_rax + _status], STS_COMPLETED_HMAC + mov qword [lane_data + _job_in_lane], 0 + + mov unused_lanes, [state + _unused_lanes] + shl unused_lanes, 4 + or unused_lanes, idx + mov [state + _unused_lanes], unused_lanes + + sub dword [state + _num_lanes_inuse_sha1], 1 + + mov p, [job_rax + _auth_tag_output] + + vzeroupper + + ; copy 12 bytes + mov DWORD(tmp), [state + _args_digest + SHA1_DIGEST_WORD_SIZE*idx + 0*SHA1_DIGEST_ROW_SIZE] + mov DWORD(tmp2), [state + _args_digest + SHA1_DIGEST_WORD_SIZE*idx + 1*SHA1_DIGEST_ROW_SIZE] + mov DWORD(tmp3), [state + _args_digest + SHA1_DIGEST_WORD_SIZE*idx + 2*SHA1_DIGEST_ROW_SIZE] + bswap DWORD(tmp) + bswap DWORD(tmp2) + bswap DWORD(tmp3) + mov [p + 0*SHA1_DIGEST_WORD_SIZE], DWORD(tmp) + mov [p + 1*SHA1_DIGEST_WORD_SIZE], DWORD(tmp2) + mov [p + 2*SHA1_DIGEST_WORD_SIZE], DWORD(tmp3) + + cmp qword [job_rax + _auth_tag_output_len_in_bytes], 12 + je clear_ret + + ;; copy remaining 8 bytes to return 20 byte digest + mov DWORD(tmp), [state + _args_digest + SHA1_DIGEST_WORD_SIZE*idx + 3*SHA1_DIGEST_ROW_SIZE] + mov DWORD(tmp2), [state + _args_digest + SHA1_DIGEST_WORD_SIZE*idx + 4*SHA1_DIGEST_ROW_SIZE] + bswap DWORD(tmp) + bswap DWORD(tmp2) + mov [p + 3*SHA1_DIGEST_WORD_SIZE], DWORD(tmp) + mov [p + 4*SHA1_DIGEST_WORD_SIZE], DWORD(tmp2) + +clear_ret: + +%ifdef SAFE_DATA + ;; Clear digest (20B), outer_block (20B) and extra_block (64B) of returned job + mov dword [state + _args_digest + SHA1_DIGEST_WORD_SIZE*idx + 0*SHA1_DIGEST_ROW_SIZE], 0 + mov dword [state + _args_digest + SHA1_DIGEST_WORD_SIZE*idx + 1*SHA1_DIGEST_ROW_SIZE], 0 + mov dword [state + _args_digest + SHA1_DIGEST_WORD_SIZE*idx + 2*SHA1_DIGEST_ROW_SIZE], 0 + mov dword [state + _args_digest + SHA1_DIGEST_WORD_SIZE*idx + 3*SHA1_DIGEST_ROW_SIZE], 0 + mov dword [state + _args_digest + SHA1_DIGEST_WORD_SIZE*idx + 4*SHA1_DIGEST_ROW_SIZE], 0 + + vpxorq zmm0, zmm0 + imul lane_data, idx, _HMAC_SHA1_LANE_DATA_size + lea lane_data, [state + _ldata + lane_data] + + ;; Clear first 64 bytes of extra_block + vmovdqu64 [lane_data + _extra_block], zmm0 + + ;; Clear first 20 bytes of outer_block + vmovdqu64 [lane_data + _outer_block], xmm0 + mov dword [lane_data + _outer_block + 16], 0 +%endif + +return: + DBGPRINTL "---------- exit sha1 submit -----------" + mov rbp, [rsp + _gpr_save + 8*0] + mov r12, [rsp + _gpr_save + 8*1] + mov r13, [rsp + _gpr_save + 8*2] + mov r14, [rsp + _gpr_save + 8*3] + mov r15, [rsp + _gpr_save + 8*4] +%ifndef LINUX + mov rsi, [rsp + _gpr_save + 8*5] + mov rdi, [rsp + _gpr_save + 8*6] +%endif + mov rsp, [rsp + _rsp_save] + + ret + +%ifdef LINUX +section .note.GNU-stack noalloc noexec nowrite progbits +%endif diff --git a/src/spdk/intel-ipsec-mb/avx512/sha1_x16_avx512.asm b/src/spdk/intel-ipsec-mb/avx512/sha1_x16_avx512.asm new file mode 100644 index 000000000..d67046ce5 --- /dev/null +++ b/src/spdk/intel-ipsec-mb/avx512/sha1_x16_avx512.asm @@ -0,0 +1,439 @@ +;; +;; Copyright (c) 2012-2018, Intel Corporation +;; +;; Redistribution and use in source and binary forms, with or without +;; modification, are permitted provided that the following conditions are met: +;; +;; * Redistributions of source code must retain the above copyright notice, +;; this list of conditions and the following disclaimer. +;; * Redistributions in binary form must reproduce the above copyright +;; notice, this list of conditions and the following disclaimer in the +;; documentation and/or other materials provided with the distribution. +;; * Neither the name of Intel Corporation nor the names of its contributors +;; may be used to endorse or promote products derived from this software +;; without specific prior written permission. +;; +;; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +;; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +;; IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +;; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE +;; FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +;; DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +;; SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +;; CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +;; OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +;; + +;; Stack must be aligned to 32 bytes before call +;; +;; Registers: RAX RBX RCX RDX RBP RSI RDI R8 R9 R10 R11 R12 R13 R14 R15 +;; ----------------------------------------------------------- +;; Windows clobbers: RAX RDX R8 R9 R10 R11 R12 R13 R14 R15 +;; Windows preserves: RBX RCX RBP RSI RDI +;; ----------------------------------------------------------- +;; Linux clobbers: RAX RDX RSI R9 R10 R11 R12 R13 R14 R15 +;; Linux preserves: RBX RCX RBP RDI R8 +;; ----------------------------------------------------------- +;; Clobbers ZMM0-31 + +%include "include/os.asm" +;%define DO_DBGPRINT +%include "include/dbgprint.asm" +%include "mb_mgr_datastruct.asm" +%include "include/transpose_avx512.asm" +%include "include/reg_sizes.asm" + +section .data +default rel +align 64 +K00_19: ;ddq 0x5A8279995A8279995A8279995A827999 + ;ddq 0x5A8279995A8279995A8279995A827999 + ;ddq 0x5A8279995A8279995A8279995A827999 + ;ddq 0x5A8279995A8279995A8279995A827999 + dq 0x5A8279995A827999, 0x5A8279995A827999 + dq 0x5A8279995A827999, 0x5A8279995A827999 + dq 0x5A8279995A827999, 0x5A8279995A827999 + dq 0x5A8279995A827999, 0x5A8279995A827999 +K20_39: ;ddq 0x6ED9EBA16ED9EBA16ED9EBA16ED9EBA1 + ;ddq 0x6ED9EBA16ED9EBA16ED9EBA16ED9EBA1 + ;ddq 0x6ED9EBA16ED9EBA16ED9EBA16ED9EBA1 + ;ddq 0x6ED9EBA16ED9EBA16ED9EBA16ED9EBA1 + dq 0x6ED9EBA16ED9EBA1, 0x6ED9EBA16ED9EBA1 + dq 0x6ED9EBA16ED9EBA1, 0x6ED9EBA16ED9EBA1 + dq 0x6ED9EBA16ED9EBA1, 0x6ED9EBA16ED9EBA1 + dq 0x6ED9EBA16ED9EBA1, 0x6ED9EBA16ED9EBA1 +K40_59: ;ddq 0x8F1BBCDC8F1BBCDC8F1BBCDC8F1BBCDC + ;ddq 0x8F1BBCDC8F1BBCDC8F1BBCDC8F1BBCDC + ;ddq 0x8F1BBCDC8F1BBCDC8F1BBCDC8F1BBCDC + ;ddq 0x8F1BBCDC8F1BBCDC8F1BBCDC8F1BBCDC + dq 0x8F1BBCDC8F1BBCDC, 0x8F1BBCDC8F1BBCDC + dq 0x8F1BBCDC8F1BBCDC, 0x8F1BBCDC8F1BBCDC + dq 0x8F1BBCDC8F1BBCDC, 0x8F1BBCDC8F1BBCDC + dq 0x8F1BBCDC8F1BBCDC, 0x8F1BBCDC8F1BBCDC +K60_79: ;ddq 0xCA62C1D6CA62C1D6CA62C1D6CA62C1D6 + ;ddq 0xCA62C1D6CA62C1D6CA62C1D6CA62C1D6 + ;ddq 0xCA62C1D6CA62C1D6CA62C1D6CA62C1D6 + ;ddq 0xCA62C1D6CA62C1D6CA62C1D6CA62C1D6 + dq 0xCA62C1D6CA62C1D6, 0xCA62C1D6CA62C1D6 + dq 0xCA62C1D6CA62C1D6, 0xCA62C1D6CA62C1D6 + dq 0xCA62C1D6CA62C1D6, 0xCA62C1D6CA62C1D6 + dq 0xCA62C1D6CA62C1D6, 0xCA62C1D6CA62C1D6 + +PSHUFFLE_BYTE_FLIP_MASK: ;ddq 0x0c0d0e0f08090a0b0405060700010203 + ;ddq 0x0c0d0e0f08090a0b0405060700010203 + ;ddq 0x0c0d0e0f08090a0b0405060700010203 + ;ddq 0x0c0d0e0f08090a0b0405060700010203 + dq 0x0405060700010203, 0x0c0d0e0f08090a0b + dq 0x0405060700010203, 0x0c0d0e0f08090a0b + dq 0x0405060700010203, 0x0c0d0e0f08090a0b + dq 0x0405060700010203, 0x0c0d0e0f08090a0b + +section .text + +%define APPEND(a,b) a %+ b + +%ifdef LINUX +%define arg1 rdi +%define arg2 rsi +%define arg3 rdx +%define arg4 rcx +%else +%define arg1 rcx +%define arg2 rdx +%define arg3 r8 +%define arg4 r9 +%endif + +%define state arg1 +%define SIZE arg2 +%define IDX arg3 + +%define A zmm0 +%define B zmm1 +%define C zmm2 +%define D zmm3 +%define E zmm4 +%define KT zmm5 +%define AA zmm6 +%define BB zmm7 +%define CC zmm8 +%define DD zmm9 +%define EE zmm10 +%define TMP0 zmm11 +%define TMP1 zmm12 +%define TMP2 zmm13 +%define TMP3 zmm14 +%define TMP4 zmm15 + +%define W0 zmm16 +%define W1 zmm17 +%define W2 zmm18 +%define W3 zmm19 +%define W4 zmm20 +%define W5 zmm21 +%define W6 zmm22 +%define W7 zmm23 +%define W8 zmm24 +%define W9 zmm25 +%define W10 zmm26 +%define W11 zmm27 +%define W12 zmm28 +%define W13 zmm29 +%define W14 zmm30 +%define W15 zmm31 + +%define inp0 r9 +%define inp1 r10 +%define inp2 r11 +%define inp3 r12 +%define inp4 r13 +%define inp5 r14 +%define inp6 r15 +%define inp7 rax + +%macro ROTATE_ARGS 0 +%xdefine TMP_ E +%xdefine E D +%xdefine D C +%xdefine C B +%xdefine B A +%xdefine A TMP_ +%endm + +%macro PROCESS_LOOP 2 +%define %%WT %1 +%define %%F_IMMED %2 + + ; T = ROTL_5(A) + Ft(B,C,D) + E + Kt + Wt + ; E=D, D=C, C=ROTL_30(B), B=A, A=T + + ; Ft + ; 0-19 Ch(B,C,D) = (B&C) ^ (~B&D) + ; 20-39, 60-79 Parity(B,C,D) = B ^ C ^ D + ; 40-59 Maj(B,C,D) = (B&C) ^ (B&D) ^ (C&D) + + vmovdqa32 TMP1, B ; Copy B + vpaddd E, E, %%WT ; E = E + Wt + vpternlogd TMP1, C, D, %%F_IMMED ; TMP1 = Ft(B,C,D) + vpaddd E, E, KT ; E = E + Wt + Kt + vprold TMP0, A, 5 ; TMP0 = ROTL_5(A) + vpaddd E, E, TMP1 ; E = Ft(B,C,D) + E + Kt + Wt + vprold B, B, 30 ; B = ROTL_30(B) + vpaddd E, E, TMP0 ; E = T + + ROTATE_ARGS +%endmacro + +%macro MSG_SCHED_ROUND_16_79 4 +%define %%WT %1 +%define %%WTp2 %2 +%define %%WTp8 %3 +%define %%WTp13 %4 + ; Wt = ROTL_1(Wt-3 ^ Wt-8 ^ Wt-14 ^ Wt-16) + ; Wt+16 = ROTL_1(Wt+13 ^ Wt+8 ^ Wt+2 ^ Wt) + vpternlogd %%WT, %%WTp2, %%WTp8, 0x96 + vpxord %%WT, %%WT, %%WTp13 + vprold %%WT, %%WT, 1 +%endmacro + + +; Note this is reading in two blocks of data from each lane, +; in preparation for the upcoming needed transpose to build msg schedule. +; Each register will contain 32 bytes from one lane plus 32 bytes +; from another lane. +; The first 8 registers will contain the first 32 bytes of all lanes, +; where register X (0 <= X <= 7) will contain bytes 0-31 from lane X in the first half +; and 0-31 bytes from lane X+8 in the second half. +; The last 8 registers will contain the last 32 bytes of all lanes, +; where register Y (8 <= Y <= 15) wil contain bytes 32-63 from lane Y-8 in the first half +; and 32-63 bytes from lane Y in the second half. +; This method helps reducing the number of shuffles required to transpose the data. +%macro MSG_SCHED_ROUND_00_15 6 +%define %%Wt %1 ; [out] zmm register to load the next block +%define %%LANE_IDX %2 ; [in] lane index (0-15) +%define %%BASE_PTR %3 ; [in] base address of the input data +%define %%OFFSET_PTR %4 ; [in] offset to get next block of data from the lane +%define %%TMP1 %5 ; [clobbered] temporary gp register +%define %%TMP2 %6 ; [clobbered] temporary gp register +%if (%%LANE_IDX < 8) + mov %%TMP1, [%%BASE_PTR + %%LANE_IDX*PTR_SZ] + mov %%TMP2, [%%BASE_PTR + (%%LANE_IDX+8)*PTR_SZ] + vmovups YWORD(%%Wt), [%%TMP1+%%OFFSET_PTR] + vinserti64x4 %%Wt, %%Wt, [%%TMP2+%%OFFSET_PTR], 0x01 +%else + mov %%TMP1, [%%BASE_PTR + (%%LANE_IDX-8)*PTR_SZ] + mov %%TMP2, [%%BASE_PTR + %%LANE_IDX*PTR_SZ] + vmovups YWORD(%%Wt), [%%TMP1+%%OFFSET_PTR+32] + vinserti64x4 %%Wt, %%Wt, [%%TMP2+%%OFFSET_PTR+32], 0x01 +%endif +%endmacro + +align 64 +; void sha1_mult_x16_avx3(void **input_data, UINT128 *digest, UINT32 size) +; arg 1 : pointer to SHA1 args structure +; arg 2 : size (in blocks) ;; assumed to be >= 1 +MKGLOBAL(sha1_x16_avx512,function,internal) +sha1_x16_avx512: + ;; Initialize digests + vmovdqu32 A, [state + 0*SHA1_DIGEST_ROW_SIZE] + vmovdqu32 B, [state + 1*SHA1_DIGEST_ROW_SIZE] + vmovdqu32 C, [state + 2*SHA1_DIGEST_ROW_SIZE] + vmovdqu32 D, [state + 3*SHA1_DIGEST_ROW_SIZE] + vmovdqu32 E, [state + 4*SHA1_DIGEST_ROW_SIZE] + DBGPRINTL_ZMM "Sha1-AVX512 incoming transposed digest", A, B, C, D, E + DBGPRINTL64 "SIZE", SIZE + + xor IDX, IDX + + ;; Load first blocks of data into ZMM registers before + ;; performing a 16x16 32-bit transpose. + ;; To speed up the transpose, data is loaded in chunks of 32 bytes, + ;; interleaving data between lane X and lane X+8. + ;; This way, final shuffles between top half and bottom half + ;; of the matrix are avoided. + mov inp0, [state + _data_ptr_sha1 + 0*PTR_SZ] + mov inp1, [state + _data_ptr_sha1 + 1*PTR_SZ] + mov inp2, [state + _data_ptr_sha1 + 2*PTR_SZ] + mov inp3, [state + _data_ptr_sha1 + 3*PTR_SZ] + mov inp4, [state + _data_ptr_sha1 + 4*PTR_SZ] + mov inp5, [state + _data_ptr_sha1 + 5*PTR_SZ] + mov inp6, [state + _data_ptr_sha1 + 6*PTR_SZ] + mov inp7, [state + _data_ptr_sha1 + 7*PTR_SZ] + + TRANSPOSE16_U32_LOAD_FIRST8 W0, W1, W2, W3, W4, W5, W6, W7, \ + W8, W9, W10, W11, W12, W13, W14, W15, \ + inp0, inp1, inp2, inp3, inp4, inp5, \ + inp6, inp7, IDX + + mov inp0, [state + _data_ptr_sha1 + 8*PTR_SZ] + mov inp1, [state + _data_ptr_sha1 + 9*PTR_SZ] + mov inp2, [state + _data_ptr_sha1 +10*PTR_SZ] + mov inp3, [state + _data_ptr_sha1 +11*PTR_SZ] + mov inp4, [state + _data_ptr_sha1 +12*PTR_SZ] + mov inp5, [state + _data_ptr_sha1 +13*PTR_SZ] + mov inp6, [state + _data_ptr_sha1 +14*PTR_SZ] + mov inp7, [state + _data_ptr_sha1 +15*PTR_SZ] + + TRANSPOSE16_U32_LOAD_LAST8 W0, W1, W2, W3, W4, W5, W6, W7, \ + W8, W9, W10, W11, W12, W13, W14, W15, \ + inp0, inp1, inp2, inp3, inp4, inp5, \ + inp6, inp7, IDX +lloop: + vmovdqa32 TMP2, [rel PSHUFFLE_BYTE_FLIP_MASK] + + add IDX, 64 + + TRANSPOSE16_U32 W0, W1, W2, W3, W4, W5, W6, W7, W8, W9, W10, W11, W12, W13, W14, W15, TMP0, TMP1, TMP3, TMP4 + DBGPRINTL_ZMM "Sha1-AVX512 incoming transposed input", W0, W1, W2, W3, W4, W6, W7, W8, W9, W10, W11, W12, W13, W14, W15 + +%assign I 0 +%rep 16 + vpshufb APPEND(W,I), APPEND(W,I), TMP2 +%assign I (I+1) +%endrep + + ; Save digests for later addition + vmovdqa32 AA, A + vmovdqa32 BB, B + vmovdqa32 CC, C + vmovdqa32 DD, D + vmovdqa32 EE, E + + vmovdqa32 KT, [rel K00_19] +%assign I 0xCA +%assign J 0 +%assign K 2 +%assign L 8 +%assign M 13 +%assign N 0 +%rep 64 + PROCESS_LOOP APPEND(W,J), I + MSG_SCHED_ROUND_16_79 APPEND(W,J), APPEND(W,K), APPEND(W,L), APPEND(W,M) + %if N = 19 + vmovdqa32 KT, [rel K20_39] + %assign I 0x96 + %elif N = 39 + vmovdqa32 KT, [rel K40_59] + %assign I 0xE8 + %elif N = 59 + vmovdqa32 KT, [rel K60_79] + %assign I 0x96 + %endif +%assign J ((J+1)% 16) +%assign K ((K+1)% 16) +%assign L ((L+1)% 16) +%assign M ((M+1)% 16) +%assign N (N+1) +%endrep + + ; Check if this is the last block + sub SIZE, 1 + je lastLoop + +%assign I 0x96 +%assign J 0 +%rep 16 + PROCESS_LOOP APPEND(W,J), I + MSG_SCHED_ROUND_00_15 APPEND(W,J), J, state + _data_ptr_sha1, IDX, inp0, inp1 +%assign J (J+1) +%endrep + + ; Add old digest + vpaddd A,A,AA + vpaddd B,B,BB + vpaddd C,C,CC + vpaddd D,D,DD + vpaddd E,E,EE + + jmp lloop + +lastLoop: +; Need to reset argument rotation values to Round 64 values +%xdefine TMP_ A +%xdefine A B +%xdefine B C +%xdefine C D +%xdefine D E +%xdefine E TMP_ + + ; Process last 16 rounds +%assign I 0x96 +%assign J 0 +%rep 16 + PROCESS_LOOP APPEND(W,J), I +%assign J (J+1) +%endrep + + ; Add old digest + vpaddd A,A,AA + vpaddd B,B,BB + vpaddd C,C,CC + vpaddd D,D,DD + vpaddd E,E,EE + + ; Write out digest + ; Do we need to untranspose digests??? + vmovdqu32 [state + 0*SHA1_DIGEST_ROW_SIZE], A + vmovdqu32 [state + 1*SHA1_DIGEST_ROW_SIZE], B + vmovdqu32 [state + 2*SHA1_DIGEST_ROW_SIZE], C + vmovdqu32 [state + 3*SHA1_DIGEST_ROW_SIZE], D + vmovdqu32 [state + 4*SHA1_DIGEST_ROW_SIZE], E + DBGPRINTL_ZMM "Sha1-AVX512 outgoing transposed digest", A, B, C, D, E + + ;; update input pointers + mov inp0, [state + _data_ptr_sha1 + 0*PTR_SZ] + mov inp1, [state + _data_ptr_sha1 + 1*PTR_SZ] + mov inp2, [state + _data_ptr_sha1 + 2*PTR_SZ] + mov inp3, [state + _data_ptr_sha1 + 3*PTR_SZ] + mov inp4, [state + _data_ptr_sha1 + 4*PTR_SZ] + mov inp5, [state + _data_ptr_sha1 + 5*PTR_SZ] + mov inp6, [state + _data_ptr_sha1 + 6*PTR_SZ] + mov inp7, [state + _data_ptr_sha1 + 7*PTR_SZ] + add inp0, IDX + add inp1, IDX + add inp2, IDX + add inp3, IDX + add inp4, IDX + add inp5, IDX + add inp6, IDX + add inp7, IDX + mov [state + _data_ptr_sha1 + 0*PTR_SZ], inp0 + mov [state + _data_ptr_sha1 + 1*PTR_SZ], inp1 + mov [state + _data_ptr_sha1 + 2*PTR_SZ], inp2 + mov [state + _data_ptr_sha1 + 3*PTR_SZ], inp3 + mov [state + _data_ptr_sha1 + 4*PTR_SZ], inp4 + mov [state + _data_ptr_sha1 + 5*PTR_SZ], inp5 + mov [state + _data_ptr_sha1 + 6*PTR_SZ], inp6 + mov [state + _data_ptr_sha1 + 7*PTR_SZ], inp7 + + mov inp0, [state + _data_ptr_sha1 + 8*PTR_SZ] + mov inp1, [state + _data_ptr_sha1 + 9*PTR_SZ] + mov inp2, [state + _data_ptr_sha1 + 10*PTR_SZ] + mov inp3, [state + _data_ptr_sha1 + 11*PTR_SZ] + mov inp4, [state + _data_ptr_sha1 + 12*PTR_SZ] + mov inp5, [state + _data_ptr_sha1 + 13*PTR_SZ] + mov inp6, [state + _data_ptr_sha1 + 14*PTR_SZ] + mov inp7, [state + _data_ptr_sha1 + 15*PTR_SZ] + add inp0, IDX + add inp1, IDX + add inp2, IDX + add inp3, IDX + add inp4, IDX + add inp5, IDX + add inp6, IDX + add inp7, IDX + mov [state + _data_ptr_sha1 + 8*PTR_SZ], inp0 + mov [state + _data_ptr_sha1 + 9*PTR_SZ], inp1 + mov [state + _data_ptr_sha1 + 10*PTR_SZ], inp2 + mov [state + _data_ptr_sha1 + 11*PTR_SZ], inp3 + mov [state + _data_ptr_sha1 + 12*PTR_SZ], inp4 + mov [state + _data_ptr_sha1 + 13*PTR_SZ], inp5 + mov [state + _data_ptr_sha1 + 14*PTR_SZ], inp6 + mov [state + _data_ptr_sha1 + 15*PTR_SZ], inp7 + + ret + +%ifdef LINUX +section .note.GNU-stack noalloc noexec nowrite progbits +%endif diff --git a/src/spdk/intel-ipsec-mb/avx512/sha256_x16_avx512.asm b/src/spdk/intel-ipsec-mb/avx512/sha256_x16_avx512.asm new file mode 100644 index 000000000..cdbb61ea3 --- /dev/null +++ b/src/spdk/intel-ipsec-mb/avx512/sha256_x16_avx512.asm @@ -0,0 +1,758 @@ +;; +;; Copyright (c) 2017-2018, Intel Corporation +;; +;; Redistribution and use in source and binary forms, with or without +;; modification, are permitted provided that the following conditions are met: +;; +;; * Redistributions of source code must retain the above copyright notice, +;; this list of conditions and the following disclaimer. +;; * Redistributions in binary form must reproduce the above copyright +;; notice, this list of conditions and the following disclaimer in the +;; documentation and/or other materials provided with the distribution. +;; * Neither the name of Intel Corporation nor the names of its contributors +;; may be used to endorse or promote products derived from this software +;; without specific prior written permission. +;; +;; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +;; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +;; IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +;; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE +;; FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +;; DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +;; SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +;; CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +;; OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +;; + +;; Stack must be aligned to 32 bytes before call +;; +;; Registers: RAX RBX RCX RDX RBP RSI RDI R8 R9 R10 R11 R12 R13 R14 R15 +;; ----------------------------------------------------------- +;; Windows clobbers: RAX RDX RSI RDI R9 R10 R11 R12 R13 R14 R15 +;; Windows preserves: RCX +;; ----------------------------------------------------------- +;; Linux clobbers: RAX RCX RDX RSI R9 R10 R11 R12 R13 R14 R15 +;; Linux preserves: RDI +;; ----------------------------------------------------------- +;; Clobbers ZMM0-31 + +%include "include/os.asm" +;%define DO_DBGPRINT +%include "include/dbgprint.asm" +%include "mb_mgr_datastruct.asm" +%include "include/transpose_avx512.asm" +%include "include/reg_sizes.asm" + +; re-use K256 from sha256_oct_avx2.asm +extern K256 + +;; code to compute x16 SHA256 using AVX512 + +%define APPEND(a,b) a %+ b + +; Define Stack Layout +START_FIELDS +;;; name size align +FIELD _DIGEST_SAVE, 8*64, 64 +FIELD _rsp, 8, 8 +%assign STACK_SPACE _FIELD_OFFSET + +%ifdef LINUX +; Linux register definitions + %define arg1 rdi + %define arg2 rsi + %define arg3 rcx + %define arg4 rdx +%else +; Windows definitions + %define arg1 rcx + %define arg2 rdx + %define arg3 rsi + %define arg4 rdi +%endif + +%define STATE arg1 +%define INP_SIZE arg2 +%define IDX arg3 +%define TBL arg4 + +%define A zmm0 +%define B zmm1 +%define C zmm2 +%define D zmm3 +%define E zmm4 +%define F zmm5 +%define G zmm6 +%define H zmm7 +%define T1 zmm8 +%define TMP0 zmm9 +%define TMP1 zmm10 +%define TMP2 zmm11 +%define TMP3 zmm12 +%define TMP4 zmm13 +%define TMP5 zmm14 +%define TMP6 zmm15 + +%define W0 zmm16 +%define W1 zmm17 +%define W2 zmm18 +%define W3 zmm19 +%define W4 zmm20 +%define W5 zmm21 +%define W6 zmm22 +%define W7 zmm23 +%define W8 zmm24 +%define W9 zmm25 +%define W10 zmm26 +%define W11 zmm27 +%define W12 zmm28 +%define W13 zmm29 +%define W14 zmm30 +%define W15 zmm31 + +%define inp0 r9 +%define inp1 r10 +%define inp2 r11 +%define inp3 r12 +%define inp4 r13 +%define inp5 r14 +%define inp6 r15 +%define inp7 rax + +%macro ROTATE_ARGS 0 +%xdefine TMP_ H +%xdefine H G +%xdefine G F +%xdefine F E +%xdefine E D +%xdefine D C +%xdefine C B +%xdefine B A +%xdefine A TMP_ +%endm + +;; CH(A, B, C) = (A&B) ^ (~A&C) +;; MAJ(E, F, G) = (E&F) ^ (E&G) ^ (F&G) +;; SIGMA0 = ROR_2 ^ ROR_13 ^ ROR_22 +;; SIGMA1 = ROR_6 ^ ROR_11 ^ ROR_25 +;; sigma0 = ROR_7 ^ ROR_18 ^ SHR_3 +;; sigma1 = ROR_17 ^ ROR_19 ^ SHR_10 + +; Main processing loop per round +%macro PROCESS_LOOP 2 +%define %%WT %1 +%define %%ROUND %2 + ;; T1 = H + SIGMA1(E) + CH(E, F, G) + Kt + Wt + ;; T2 = SIGMA0(A) + MAJ(A, B, C) + ;; H=G, G=F, F=E, E=D+T1, D=C, C=B, B=A, A=T1+T2 + + ;; H becomes T2, then add T1 for A + ;; D becomes D + T1 for E + + vpaddd T1, H, TMP3 ; T1 = H + Kt + vmovdqa32 TMP0, E + vprord TMP1, E, 6 ; ROR_6(E) + vprord TMP2, E, 11 ; ROR_11(E) + vprord TMP3, E, 25 ; ROR_25(E) + vpternlogd TMP0, F, G, 0xCA ; TMP0 = CH(E,F,G) + vpaddd T1, T1, %%WT ; T1 = T1 + Wt + vpternlogd TMP1, TMP2, TMP3, 0x96 ; TMP1 = SIGMA1(E) + vpaddd T1, T1, TMP0 ; T1 = T1 + CH(E,F,G) + vpaddd T1, T1, TMP1 ; T1 = T1 + SIGMA1(E) + vpaddd D, D, T1 ; D = D + T1 + + vprord H, A, 2 ; ROR_2(A) + vprord TMP2, A, 13 ; ROR_13(A) + vprord TMP3, A, 22 ; ROR_22(A) + vmovdqa32 TMP0, A + vpternlogd TMP0, B, C, 0xE8 ; TMP0 = MAJ(A,B,C) + vpternlogd H, TMP2, TMP3, 0x96 ; H(T2) = SIGMA0(A) + vpaddd H, H, TMP0 ; H(T2) = SIGMA0(A) + MAJ(A,B,C) + vpaddd H, H, T1 ; H(A) = H(T2) + T1 + + vmovdqa32 TMP3, [TBL + ((%%ROUND+1)*64)] ; Next Kt + + ;; Rotate the args A-H (rotation of names associated with regs) + ROTATE_ARGS +%endmacro + +; This is supposed to be SKL optimized assuming: +; vpternlog, vpaddd ports 5,8 +; vprord ports 1,8 +; However, vprord is only working on port 8 +; +; Main processing loop per round +; Get the msg schedule word 16 from the current, now unneccessary word +%macro PROCESS_LOOP_00_47 5 +%define %%WT %1 +%define %%ROUND %2 +%define %%WTp1 %3 +%define %%WTp9 %4 +%define %%WTp14 %5 + ;; T1 = H + SIGMA1(E) + CH(E, F, G) + Kt + Wt + ;; T2 = SIGMA0(A) + MAJ(A, B, C) + ;; H=G, G=F, F=E, E=D+T1, D=C, C=B, B=A, A=T1+T2 + + ;; H becomes T2, then add T1 for A + ;; D becomes D + T1 for E + + ;; For next value in msg schedule + ;; Wt+16 = sigma1(Wt+14) + Wt+9 + sigma0(Wt+1) + Wt + + vmovdqa32 TMP0, E + vprord TMP1, E, 6 ; ROR_6(E) + vprord TMP2, E, 11 ; ROR_11(E) + vprord TMP3, E, 25 ; ROR_25(E) + vpternlogd TMP0, F, G, 0xCA ; TMP0 = CH(E,F,G) + vpaddd T1, H, %%WT ; T1 = H + Wt + vpternlogd TMP1, TMP2, TMP3, 0x96 ; TMP1 = SIGMA1(E) + vpaddd T1, T1, TMP6 ; T1 = T1 + Kt + vprord H, A, 2 ; ROR_2(A) + vpaddd T1, T1, TMP0 ; T1 = T1 + CH(E,F,G) + vprord TMP2, A, 13 ; ROR_13(A) + vmovdqa32 TMP0, A + vprord TMP3, A, 22 ; ROR_22(A) + vpaddd T1, T1, TMP1 ; T1 = T1 + SIGMA1(E) + vpternlogd TMP0, B, C, 0xE8 ; TMP0 = MAJ(A,B,C) + vpaddd D, D, T1 ; D = D + T1 + vpternlogd H, TMP2, TMP3, 0x96 ; H(T2) = SIGMA0(A) + vprord TMP4, %%WTp14, 17 ; ROR_17(Wt-2) + vpaddd H, H, TMP0 ; H(T2) = SIGMA0(A) + MAJ(A,B,C) + vprord TMP5, %%WTp14, 19 ; ROR_19(Wt-2) + vpsrld TMP6, %%WTp14, 10 ; SHR_10(Wt-2) + vpaddd H, H, T1 ; H(A) = H(T2) + T1 + vpternlogd TMP4, TMP5, TMP6, 0x96 ; TMP4 = sigma1(Wt-2) + vpaddd %%WT, %%WT, TMP4 ; Wt = Wt-16 + sigma1(Wt-2) + vprord TMP4, %%WTp1, 7 ; ROR_7(Wt-15) + vprord TMP5, %%WTp1, 18 ; ROR_18(Wt-15) + vpaddd %%WT, %%WT, %%WTp9 ; Wt = Wt-16 + sigma1(Wt-2) + Wt-7 + vpsrld TMP6, %%WTp1, 3 ; SHR_3(Wt-15) + vpternlogd TMP4, TMP5, TMP6, 0x96 ; TMP4 = sigma0(Wt-15) + vpaddd %%WT, %%WT, TMP4 ; Wt = Wt-16 + sigma1(Wt-2) + + ; Wt-7 + sigma0(Wt-15) + + + vmovdqa32 TMP6, [TBL + ((%%ROUND+1)*64)] ; Next Kt + + ;; Rotate the args A-H (rotation of names associated with regs) + ROTATE_ARGS +%endmacro + +%macro MSG_SCHED_ROUND_16_63 4 +%define %%WT %1 +%define %%WTp1 %2 +%define %%WTp9 %3 +%define %%WTp14 %4 + vprord TMP4, %%WTp14, 17 ; ROR_17(Wt-2) + vprord TMP5, %%WTp14, 19 ; ROR_19(Wt-2) + vpsrld TMP6, %%WTp14, 10 ; SHR_10(Wt-2) + vpternlogd TMP4, TMP5, TMP6, 0x96 ; TMP4 = sigma1(Wt-2) + + vpaddd %%WT, %%WT, TMP4 ; Wt = Wt-16 + sigma1(Wt-2) + vpaddd %%WT, %%WT, %%WTp9 ; Wt = Wt-16 + sigma1(Wt-2) + Wt-7 + + vprord TMP4, %%WTp1, 7 ; ROR_7(Wt-15) + vprord TMP5, %%WTp1, 18 ; ROR_18(Wt-15) + vpsrld TMP6, %%WTp1, 3 ; SHR_3(Wt-15) + vpternlogd TMP4, TMP5, TMP6, 0x96 ; TMP4 = sigma0(Wt-15) + + vpaddd %%WT, %%WT, TMP4 ; Wt = Wt-16 + sigma1(Wt-2) + + ; Wt-7 + sigma0(Wt-15) + +%endmacro + +; Note this is reading in two blocks of data from each lane, +; in preparation for the upcoming needed transpose to build msg schedule. +; Each register will contain 32 bytes from one lane plus 32 bytes +; from another lane. +; The first 8 registers will contain the first 32 bytes of all lanes, +; where register X (0 <= X <= 7) will contain bytes 0-31 from lane X in the first half +; and 0-31 bytes from lane X+8 in the second half. +; The last 8 registers will contain the last 32 bytes of all lanes, +; where register Y (8 <= Y <= 15) wil contain bytes 32-63 from lane Y-8 in the first half +; and 32-63 bytes from lane Y in the second half. +; This method helps reducing the number of shuffles required to transpose the data. +%macro MSG_SCHED_ROUND_00_15 6 +%define %%Wt %1 ; [out] zmm register to load the next block +%define %%LANE_IDX %2 ; [in] lane index (0-15) +%define %%BASE_PTR %3 ; [in] base address of the input data +%define %%OFFSET_PTR %4 ; [in] offset to get next block of data from the lane +%define %%TMP1 %5 ; [clobbered] temporary gp register +%define %%TMP2 %6 ; [clobbered] temporary gp register +%if (%%LANE_IDX < 8) + mov %%TMP1, [%%BASE_PTR + %%LANE_IDX*PTR_SZ] + mov %%TMP2, [%%BASE_PTR + (%%LANE_IDX+8)*PTR_SZ] + vmovups YWORD(%%Wt), [%%TMP1+%%OFFSET_PTR] + vinserti64x4 %%Wt, %%Wt, [%%TMP2+%%OFFSET_PTR], 0x01 +%else + mov %%TMP1, [%%BASE_PTR + (%%LANE_IDX-8)*PTR_SZ] + mov %%TMP2, [%%BASE_PTR + %%LANE_IDX*PTR_SZ] + vmovups YWORD(%%Wt), [%%TMP1+%%OFFSET_PTR+32] + vinserti64x4 %%Wt, %%Wt, [%%TMP2+%%OFFSET_PTR+32], 0x01 +%endif +%endmacro + + section .data +default rel +align 64 +TABLE: + dq 0x428a2f98428a2f98, 0x428a2f98428a2f98 + dq 0x428a2f98428a2f98, 0x428a2f98428a2f98 + dq 0x428a2f98428a2f98, 0x428a2f98428a2f98 + dq 0x428a2f98428a2f98, 0x428a2f98428a2f98 + dq 0x7137449171374491, 0x7137449171374491 + dq 0x7137449171374491, 0x7137449171374491 + dq 0x7137449171374491, 0x7137449171374491 + dq 0x7137449171374491, 0x7137449171374491 + dq 0xb5c0fbcfb5c0fbcf, 0xb5c0fbcfb5c0fbcf + dq 0xb5c0fbcfb5c0fbcf, 0xb5c0fbcfb5c0fbcf + dq 0xb5c0fbcfb5c0fbcf, 0xb5c0fbcfb5c0fbcf + dq 0xb5c0fbcfb5c0fbcf, 0xb5c0fbcfb5c0fbcf + dq 0xe9b5dba5e9b5dba5, 0xe9b5dba5e9b5dba5 + dq 0xe9b5dba5e9b5dba5, 0xe9b5dba5e9b5dba5 + dq 0xe9b5dba5e9b5dba5, 0xe9b5dba5e9b5dba5 + dq 0xe9b5dba5e9b5dba5, 0xe9b5dba5e9b5dba5 + dq 0x3956c25b3956c25b, 0x3956c25b3956c25b + dq 0x3956c25b3956c25b, 0x3956c25b3956c25b + dq 0x3956c25b3956c25b, 0x3956c25b3956c25b + dq 0x3956c25b3956c25b, 0x3956c25b3956c25b + dq 0x59f111f159f111f1, 0x59f111f159f111f1 + dq 0x59f111f159f111f1, 0x59f111f159f111f1 + dq 0x59f111f159f111f1, 0x59f111f159f111f1 + dq 0x59f111f159f111f1, 0x59f111f159f111f1 + dq 0x923f82a4923f82a4, 0x923f82a4923f82a4 + dq 0x923f82a4923f82a4, 0x923f82a4923f82a4 + dq 0x923f82a4923f82a4, 0x923f82a4923f82a4 + dq 0x923f82a4923f82a4, 0x923f82a4923f82a4 + dq 0xab1c5ed5ab1c5ed5, 0xab1c5ed5ab1c5ed5 + dq 0xab1c5ed5ab1c5ed5, 0xab1c5ed5ab1c5ed5 + dq 0xab1c5ed5ab1c5ed5, 0xab1c5ed5ab1c5ed5 + dq 0xab1c5ed5ab1c5ed5, 0xab1c5ed5ab1c5ed5 + dq 0xd807aa98d807aa98, 0xd807aa98d807aa98 + dq 0xd807aa98d807aa98, 0xd807aa98d807aa98 + dq 0xd807aa98d807aa98, 0xd807aa98d807aa98 + dq 0xd807aa98d807aa98, 0xd807aa98d807aa98 + dq 0x12835b0112835b01, 0x12835b0112835b01 + dq 0x12835b0112835b01, 0x12835b0112835b01 + dq 0x12835b0112835b01, 0x12835b0112835b01 + dq 0x12835b0112835b01, 0x12835b0112835b01 + dq 0x243185be243185be, 0x243185be243185be + dq 0x243185be243185be, 0x243185be243185be + dq 0x243185be243185be, 0x243185be243185be + dq 0x243185be243185be, 0x243185be243185be + dq 0x550c7dc3550c7dc3, 0x550c7dc3550c7dc3 + dq 0x550c7dc3550c7dc3, 0x550c7dc3550c7dc3 + dq 0x550c7dc3550c7dc3, 0x550c7dc3550c7dc3 + dq 0x550c7dc3550c7dc3, 0x550c7dc3550c7dc3 + dq 0x72be5d7472be5d74, 0x72be5d7472be5d74 + dq 0x72be5d7472be5d74, 0x72be5d7472be5d74 + dq 0x72be5d7472be5d74, 0x72be5d7472be5d74 + dq 0x72be5d7472be5d74, 0x72be5d7472be5d74 + dq 0x80deb1fe80deb1fe, 0x80deb1fe80deb1fe + dq 0x80deb1fe80deb1fe, 0x80deb1fe80deb1fe + dq 0x80deb1fe80deb1fe, 0x80deb1fe80deb1fe + dq 0x80deb1fe80deb1fe, 0x80deb1fe80deb1fe + dq 0x9bdc06a79bdc06a7, 0x9bdc06a79bdc06a7 + dq 0x9bdc06a79bdc06a7, 0x9bdc06a79bdc06a7 + dq 0x9bdc06a79bdc06a7, 0x9bdc06a79bdc06a7 + dq 0x9bdc06a79bdc06a7, 0x9bdc06a79bdc06a7 + dq 0xc19bf174c19bf174, 0xc19bf174c19bf174 + dq 0xc19bf174c19bf174, 0xc19bf174c19bf174 + dq 0xc19bf174c19bf174, 0xc19bf174c19bf174 + dq 0xc19bf174c19bf174, 0xc19bf174c19bf174 + dq 0xe49b69c1e49b69c1, 0xe49b69c1e49b69c1 + dq 0xe49b69c1e49b69c1, 0xe49b69c1e49b69c1 + dq 0xe49b69c1e49b69c1, 0xe49b69c1e49b69c1 + dq 0xe49b69c1e49b69c1, 0xe49b69c1e49b69c1 + dq 0xefbe4786efbe4786, 0xefbe4786efbe4786 + dq 0xefbe4786efbe4786, 0xefbe4786efbe4786 + dq 0xefbe4786efbe4786, 0xefbe4786efbe4786 + dq 0xefbe4786efbe4786, 0xefbe4786efbe4786 + dq 0x0fc19dc60fc19dc6, 0x0fc19dc60fc19dc6 + dq 0x0fc19dc60fc19dc6, 0x0fc19dc60fc19dc6 + dq 0x0fc19dc60fc19dc6, 0x0fc19dc60fc19dc6 + dq 0x0fc19dc60fc19dc6, 0x0fc19dc60fc19dc6 + dq 0x240ca1cc240ca1cc, 0x240ca1cc240ca1cc + dq 0x240ca1cc240ca1cc, 0x240ca1cc240ca1cc + dq 0x240ca1cc240ca1cc, 0x240ca1cc240ca1cc + dq 0x240ca1cc240ca1cc, 0x240ca1cc240ca1cc + dq 0x2de92c6f2de92c6f, 0x2de92c6f2de92c6f + dq 0x2de92c6f2de92c6f, 0x2de92c6f2de92c6f + dq 0x2de92c6f2de92c6f, 0x2de92c6f2de92c6f + dq 0x2de92c6f2de92c6f, 0x2de92c6f2de92c6f + dq 0x4a7484aa4a7484aa, 0x4a7484aa4a7484aa + dq 0x4a7484aa4a7484aa, 0x4a7484aa4a7484aa + dq 0x4a7484aa4a7484aa, 0x4a7484aa4a7484aa + dq 0x4a7484aa4a7484aa, 0x4a7484aa4a7484aa + dq 0x5cb0a9dc5cb0a9dc, 0x5cb0a9dc5cb0a9dc + dq 0x5cb0a9dc5cb0a9dc, 0x5cb0a9dc5cb0a9dc + dq 0x5cb0a9dc5cb0a9dc, 0x5cb0a9dc5cb0a9dc + dq 0x5cb0a9dc5cb0a9dc, 0x5cb0a9dc5cb0a9dc + dq 0x76f988da76f988da, 0x76f988da76f988da + dq 0x76f988da76f988da, 0x76f988da76f988da + dq 0x76f988da76f988da, 0x76f988da76f988da + dq 0x76f988da76f988da, 0x76f988da76f988da + dq 0x983e5152983e5152, 0x983e5152983e5152 + dq 0x983e5152983e5152, 0x983e5152983e5152 + dq 0x983e5152983e5152, 0x983e5152983e5152 + dq 0x983e5152983e5152, 0x983e5152983e5152 + dq 0xa831c66da831c66d, 0xa831c66da831c66d + dq 0xa831c66da831c66d, 0xa831c66da831c66d + dq 0xa831c66da831c66d, 0xa831c66da831c66d + dq 0xa831c66da831c66d, 0xa831c66da831c66d + dq 0xb00327c8b00327c8, 0xb00327c8b00327c8 + dq 0xb00327c8b00327c8, 0xb00327c8b00327c8 + dq 0xb00327c8b00327c8, 0xb00327c8b00327c8 + dq 0xb00327c8b00327c8, 0xb00327c8b00327c8 + dq 0xbf597fc7bf597fc7, 0xbf597fc7bf597fc7 + dq 0xbf597fc7bf597fc7, 0xbf597fc7bf597fc7 + dq 0xbf597fc7bf597fc7, 0xbf597fc7bf597fc7 + dq 0xbf597fc7bf597fc7, 0xbf597fc7bf597fc7 + dq 0xc6e00bf3c6e00bf3, 0xc6e00bf3c6e00bf3 + dq 0xc6e00bf3c6e00bf3, 0xc6e00bf3c6e00bf3 + dq 0xc6e00bf3c6e00bf3, 0xc6e00bf3c6e00bf3 + dq 0xc6e00bf3c6e00bf3, 0xc6e00bf3c6e00bf3 + dq 0xd5a79147d5a79147, 0xd5a79147d5a79147 + dq 0xd5a79147d5a79147, 0xd5a79147d5a79147 + dq 0xd5a79147d5a79147, 0xd5a79147d5a79147 + dq 0xd5a79147d5a79147, 0xd5a79147d5a79147 + dq 0x06ca635106ca6351, 0x06ca635106ca6351 + dq 0x06ca635106ca6351, 0x06ca635106ca6351 + dq 0x06ca635106ca6351, 0x06ca635106ca6351 + dq 0x06ca635106ca6351, 0x06ca635106ca6351 + dq 0x1429296714292967, 0x1429296714292967 + dq 0x1429296714292967, 0x1429296714292967 + dq 0x1429296714292967, 0x1429296714292967 + dq 0x1429296714292967, 0x1429296714292967 + dq 0x27b70a8527b70a85, 0x27b70a8527b70a85 + dq 0x27b70a8527b70a85, 0x27b70a8527b70a85 + dq 0x27b70a8527b70a85, 0x27b70a8527b70a85 + dq 0x27b70a8527b70a85, 0x27b70a8527b70a85 + dq 0x2e1b21382e1b2138, 0x2e1b21382e1b2138 + dq 0x2e1b21382e1b2138, 0x2e1b21382e1b2138 + dq 0x2e1b21382e1b2138, 0x2e1b21382e1b2138 + dq 0x2e1b21382e1b2138, 0x2e1b21382e1b2138 + dq 0x4d2c6dfc4d2c6dfc, 0x4d2c6dfc4d2c6dfc + dq 0x4d2c6dfc4d2c6dfc, 0x4d2c6dfc4d2c6dfc + dq 0x4d2c6dfc4d2c6dfc, 0x4d2c6dfc4d2c6dfc + dq 0x4d2c6dfc4d2c6dfc, 0x4d2c6dfc4d2c6dfc + dq 0x53380d1353380d13, 0x53380d1353380d13 + dq 0x53380d1353380d13, 0x53380d1353380d13 + dq 0x53380d1353380d13, 0x53380d1353380d13 + dq 0x53380d1353380d13, 0x53380d1353380d13 + dq 0x650a7354650a7354, 0x650a7354650a7354 + dq 0x650a7354650a7354, 0x650a7354650a7354 + dq 0x650a7354650a7354, 0x650a7354650a7354 + dq 0x650a7354650a7354, 0x650a7354650a7354 + dq 0x766a0abb766a0abb, 0x766a0abb766a0abb + dq 0x766a0abb766a0abb, 0x766a0abb766a0abb + dq 0x766a0abb766a0abb, 0x766a0abb766a0abb + dq 0x766a0abb766a0abb, 0x766a0abb766a0abb + dq 0x81c2c92e81c2c92e, 0x81c2c92e81c2c92e + dq 0x81c2c92e81c2c92e, 0x81c2c92e81c2c92e + dq 0x81c2c92e81c2c92e, 0x81c2c92e81c2c92e + dq 0x81c2c92e81c2c92e, 0x81c2c92e81c2c92e + dq 0x92722c8592722c85, 0x92722c8592722c85 + dq 0x92722c8592722c85, 0x92722c8592722c85 + dq 0x92722c8592722c85, 0x92722c8592722c85 + dq 0x92722c8592722c85, 0x92722c8592722c85 + dq 0xa2bfe8a1a2bfe8a1, 0xa2bfe8a1a2bfe8a1 + dq 0xa2bfe8a1a2bfe8a1, 0xa2bfe8a1a2bfe8a1 + dq 0xa2bfe8a1a2bfe8a1, 0xa2bfe8a1a2bfe8a1 + dq 0xa2bfe8a1a2bfe8a1, 0xa2bfe8a1a2bfe8a1 + dq 0xa81a664ba81a664b, 0xa81a664ba81a664b + dq 0xa81a664ba81a664b, 0xa81a664ba81a664b + dq 0xa81a664ba81a664b, 0xa81a664ba81a664b + dq 0xa81a664ba81a664b, 0xa81a664ba81a664b + dq 0xc24b8b70c24b8b70, 0xc24b8b70c24b8b70 + dq 0xc24b8b70c24b8b70, 0xc24b8b70c24b8b70 + dq 0xc24b8b70c24b8b70, 0xc24b8b70c24b8b70 + dq 0xc24b8b70c24b8b70, 0xc24b8b70c24b8b70 + dq 0xc76c51a3c76c51a3, 0xc76c51a3c76c51a3 + dq 0xc76c51a3c76c51a3, 0xc76c51a3c76c51a3 + dq 0xc76c51a3c76c51a3, 0xc76c51a3c76c51a3 + dq 0xc76c51a3c76c51a3, 0xc76c51a3c76c51a3 + dq 0xd192e819d192e819, 0xd192e819d192e819 + dq 0xd192e819d192e819, 0xd192e819d192e819 + dq 0xd192e819d192e819, 0xd192e819d192e819 + dq 0xd192e819d192e819, 0xd192e819d192e819 + dq 0xd6990624d6990624, 0xd6990624d6990624 + dq 0xd6990624d6990624, 0xd6990624d6990624 + dq 0xd6990624d6990624, 0xd6990624d6990624 + dq 0xd6990624d6990624, 0xd6990624d6990624 + dq 0xf40e3585f40e3585, 0xf40e3585f40e3585 + dq 0xf40e3585f40e3585, 0xf40e3585f40e3585 + dq 0xf40e3585f40e3585, 0xf40e3585f40e3585 + dq 0xf40e3585f40e3585, 0xf40e3585f40e3585 + dq 0x106aa070106aa070, 0x106aa070106aa070 + dq 0x106aa070106aa070, 0x106aa070106aa070 + dq 0x106aa070106aa070, 0x106aa070106aa070 + dq 0x106aa070106aa070, 0x106aa070106aa070 + dq 0x19a4c11619a4c116, 0x19a4c11619a4c116 + dq 0x19a4c11619a4c116, 0x19a4c11619a4c116 + dq 0x19a4c11619a4c116, 0x19a4c11619a4c116 + dq 0x19a4c11619a4c116, 0x19a4c11619a4c116 + dq 0x1e376c081e376c08, 0x1e376c081e376c08 + dq 0x1e376c081e376c08, 0x1e376c081e376c08 + dq 0x1e376c081e376c08, 0x1e376c081e376c08 + dq 0x1e376c081e376c08, 0x1e376c081e376c08 + dq 0x2748774c2748774c, 0x2748774c2748774c + dq 0x2748774c2748774c, 0x2748774c2748774c + dq 0x2748774c2748774c, 0x2748774c2748774c + dq 0x2748774c2748774c, 0x2748774c2748774c + dq 0x34b0bcb534b0bcb5, 0x34b0bcb534b0bcb5 + dq 0x34b0bcb534b0bcb5, 0x34b0bcb534b0bcb5 + dq 0x34b0bcb534b0bcb5, 0x34b0bcb534b0bcb5 + dq 0x34b0bcb534b0bcb5, 0x34b0bcb534b0bcb5 + dq 0x391c0cb3391c0cb3, 0x391c0cb3391c0cb3 + dq 0x391c0cb3391c0cb3, 0x391c0cb3391c0cb3 + dq 0x391c0cb3391c0cb3, 0x391c0cb3391c0cb3 + dq 0x391c0cb3391c0cb3, 0x391c0cb3391c0cb3 + dq 0x4ed8aa4a4ed8aa4a, 0x4ed8aa4a4ed8aa4a + dq 0x4ed8aa4a4ed8aa4a, 0x4ed8aa4a4ed8aa4a + dq 0x4ed8aa4a4ed8aa4a, 0x4ed8aa4a4ed8aa4a + dq 0x4ed8aa4a4ed8aa4a, 0x4ed8aa4a4ed8aa4a + dq 0x5b9cca4f5b9cca4f, 0x5b9cca4f5b9cca4f + dq 0x5b9cca4f5b9cca4f, 0x5b9cca4f5b9cca4f + dq 0x5b9cca4f5b9cca4f, 0x5b9cca4f5b9cca4f + dq 0x5b9cca4f5b9cca4f, 0x5b9cca4f5b9cca4f + dq 0x682e6ff3682e6ff3, 0x682e6ff3682e6ff3 + dq 0x682e6ff3682e6ff3, 0x682e6ff3682e6ff3 + dq 0x682e6ff3682e6ff3, 0x682e6ff3682e6ff3 + dq 0x682e6ff3682e6ff3, 0x682e6ff3682e6ff3 + dq 0x748f82ee748f82ee, 0x748f82ee748f82ee + dq 0x748f82ee748f82ee, 0x748f82ee748f82ee + dq 0x748f82ee748f82ee, 0x748f82ee748f82ee + dq 0x748f82ee748f82ee, 0x748f82ee748f82ee + dq 0x78a5636f78a5636f, 0x78a5636f78a5636f + dq 0x78a5636f78a5636f, 0x78a5636f78a5636f + dq 0x78a5636f78a5636f, 0x78a5636f78a5636f + dq 0x78a5636f78a5636f, 0x78a5636f78a5636f + dq 0x84c8781484c87814, 0x84c8781484c87814 + dq 0x84c8781484c87814, 0x84c8781484c87814 + dq 0x84c8781484c87814, 0x84c8781484c87814 + dq 0x84c8781484c87814, 0x84c8781484c87814 + dq 0x8cc702088cc70208, 0x8cc702088cc70208 + dq 0x8cc702088cc70208, 0x8cc702088cc70208 + dq 0x8cc702088cc70208, 0x8cc702088cc70208 + dq 0x8cc702088cc70208, 0x8cc702088cc70208 + dq 0x90befffa90befffa, 0x90befffa90befffa + dq 0x90befffa90befffa, 0x90befffa90befffa + dq 0x90befffa90befffa, 0x90befffa90befffa + dq 0x90befffa90befffa, 0x90befffa90befffa + dq 0xa4506ceba4506ceb, 0xa4506ceba4506ceb + dq 0xa4506ceba4506ceb, 0xa4506ceba4506ceb + dq 0xa4506ceba4506ceb, 0xa4506ceba4506ceb + dq 0xa4506ceba4506ceb, 0xa4506ceba4506ceb + dq 0xbef9a3f7bef9a3f7, 0xbef9a3f7bef9a3f7 + dq 0xbef9a3f7bef9a3f7, 0xbef9a3f7bef9a3f7 + dq 0xbef9a3f7bef9a3f7, 0xbef9a3f7bef9a3f7 + dq 0xbef9a3f7bef9a3f7, 0xbef9a3f7bef9a3f7 + dq 0xc67178f2c67178f2, 0xc67178f2c67178f2 + dq 0xc67178f2c67178f2, 0xc67178f2c67178f2 + dq 0xc67178f2c67178f2, 0xc67178f2c67178f2 + dq 0xc67178f2c67178f2, 0xc67178f2c67178f2 + +PSHUFFLE_BYTE_FLIP_MASK: + ;ddq 0x0c0d0e0f08090a0b0405060700010203 + dq 0x0405060700010203, 0x0c0d0e0f08090a0b + ;ddq 0x0c0d0e0f08090a0b0405060700010203 + dq 0x0405060700010203, 0x0c0d0e0f08090a0b + ;ddq 0x0c0d0e0f08090a0b0405060700010203 + dq 0x0405060700010203, 0x0c0d0e0f08090a0b + ;ddq 0x0c0d0e0f08090a0b0405060700010203 + dq 0x0405060700010203, 0x0c0d0e0f08090a0b + +section .text + +;; void sha256_x16_avx512(void **input_data, UINT128 *digest[16], UINT64 size) +;; arg 1 : pointer to SHA256 args structure +;; arg 2 : size (in blocks) ;; assumed to be >= 1 +;; arg 1 : rcx : pointer to array of pointers to input data +;; arg 2 : rdx : pointer to array of pointers to digest +;; arg 3 : r8 : size of input in bytes +MKGLOBAL(sha256_x16_avx512,function,internal) +align 64 +sha256_x16_avx512: + mov rax, rsp + sub rsp, STACK_SPACE + and rsp, ~63 ; align stack to multiple of 64 + mov [rsp + _rsp], rax + + ;; Initialize digests + vmovdqu32 A, [STATE + 0*SHA256_DIGEST_ROW_SIZE] + vmovdqu32 B, [STATE + 1*SHA256_DIGEST_ROW_SIZE] + vmovdqu32 C, [STATE + 2*SHA256_DIGEST_ROW_SIZE] + vmovdqu32 D, [STATE + 3*SHA256_DIGEST_ROW_SIZE] + vmovdqu32 E, [STATE + 4*SHA256_DIGEST_ROW_SIZE] + vmovdqu32 F, [STATE + 5*SHA256_DIGEST_ROW_SIZE] + vmovdqu32 G, [STATE + 6*SHA256_DIGEST_ROW_SIZE] + vmovdqu32 H, [STATE + 7*SHA256_DIGEST_ROW_SIZE] + + lea TBL, [rel TABLE] + + ; Do we need to transpose digests??? + ; SHA1 does not, but SHA256 has been + + xor IDX, IDX + + ;; Load first blocks of data into ZMM registers before + ;; performing a 16x16 32-bit transpose. + ;; To speed up the transpose, data is loaded in chunks of 32 bytes, + ;; interleaving data between lane X and lane X+8. + ;; This way, final shuffles between top half and bottom half + ;; of the matrix are avoided. + mov inp0, [STATE + _data_ptr_sha256 + 0*PTR_SZ] + mov inp1, [STATE + _data_ptr_sha256 + 1*PTR_SZ] + mov inp2, [STATE + _data_ptr_sha256 + 2*PTR_SZ] + mov inp3, [STATE + _data_ptr_sha256 + 3*PTR_SZ] + mov inp4, [STATE + _data_ptr_sha256 + 4*PTR_SZ] + mov inp5, [STATE + _data_ptr_sha256 + 5*PTR_SZ] + mov inp6, [STATE + _data_ptr_sha256 + 6*PTR_SZ] + mov inp7, [STATE + _data_ptr_sha256 + 7*PTR_SZ] + + TRANSPOSE16_U32_LOAD_FIRST8 W0, W1, W2, W3, W4, W5, W6, W7, \ + W8, W9, W10, W11, W12, W13, W14, W15, \ + inp0, inp1, inp2, inp3, inp4, inp5, \ + inp6, inp7, IDX + + mov inp0, [STATE + _data_ptr_sha256 + 8*PTR_SZ] + mov inp1, [STATE + _data_ptr_sha256 + 9*PTR_SZ] + mov inp2, [STATE + _data_ptr_sha256 +10*PTR_SZ] + mov inp3, [STATE + _data_ptr_sha256 +11*PTR_SZ] + mov inp4, [STATE + _data_ptr_sha256 +12*PTR_SZ] + mov inp5, [STATE + _data_ptr_sha256 +13*PTR_SZ] + mov inp6, [STATE + _data_ptr_sha256 +14*PTR_SZ] + mov inp7, [STATE + _data_ptr_sha256 +15*PTR_SZ] + + TRANSPOSE16_U32_LOAD_LAST8 W0, W1, W2, W3, W4, W5, W6, W7, \ + W8, W9, W10, W11, W12, W13, W14, W15, \ + inp0, inp1, inp2, inp3, inp4, inp5, \ + inp6, inp7, IDX + + align 32 +lloop: + vmovdqa32 TMP2, [rel PSHUFFLE_BYTE_FLIP_MASK] + + vmovdqa32 TMP3, [TBL] ; First K + + ; Save digests for later addition + vmovdqa32 [rsp + _DIGEST_SAVE + 64*0], A + vmovdqa32 [rsp + _DIGEST_SAVE + 64*1], B + vmovdqa32 [rsp + _DIGEST_SAVE + 64*2], C + vmovdqa32 [rsp + _DIGEST_SAVE + 64*3], D + vmovdqa32 [rsp + _DIGEST_SAVE + 64*4], E + vmovdqa32 [rsp + _DIGEST_SAVE + 64*5], F + vmovdqa32 [rsp + _DIGEST_SAVE + 64*6], G + vmovdqa32 [rsp + _DIGEST_SAVE + 64*7], H + + add IDX, 64 + + TRANSPOSE16_U32 W0, W1, W2, W3, W4, W5, W6, W7, W8, W9, W10, W11, W12, W13, W14, W15, TMP0, TMP1, TMP4, TMP5 + +%assign I 0 +%rep 16 + vpshufb APPEND(W,I), APPEND(W,I), TMP2 +%assign I (I+1) +%endrep + + ; MSG Schedule for W0-W15 is now complete in registers + ; Process first 48 rounds + ; Calculate next Wt+16 after processing is complete and Wt is unneeded + + ; PROCESS_LOOP_00_47 APPEND(W,J), I, APPEND(W,K), APPEND(W,L), APPEND(W,M) +%assign I 0 +%assign J 0 +%assign K 1 +%assign L 9 +%assign M 14 +%rep 48 + PROCESS_LOOP APPEND(W,J), I + MSG_SCHED_ROUND_16_63 APPEND(W,J), APPEND(W,K), APPEND(W,L), APPEND(W,M) +%assign I (I+1) +%assign J ((J+1)% 16) +%assign K ((K+1)% 16) +%assign L ((L+1)% 16) +%assign M ((M+1)% 16) +%endrep + + ; Check is this is the last block + sub INP_SIZE, 1 + je lastLoop + + ; Process last 16 rounds + ; Read in next block msg data for use in first 16 words of msg sched +%assign I 48 +%assign J 0 +%rep 16 + PROCESS_LOOP APPEND(W,J), I + MSG_SCHED_ROUND_00_15 APPEND(W,J), J, STATE + _data_ptr_sha256, IDX, inp0, inp1 +%assign I (I+1) +%assign J (J+1) +%endrep + + ; Add old digest + vpaddd A, A, [rsp + _DIGEST_SAVE + 64*0] + vpaddd B, B, [rsp + _DIGEST_SAVE + 64*1] + vpaddd C, C, [rsp + _DIGEST_SAVE + 64*2] + vpaddd D, D, [rsp + _DIGEST_SAVE + 64*3] + vpaddd E, E, [rsp + _DIGEST_SAVE + 64*4] + vpaddd F, F, [rsp + _DIGEST_SAVE + 64*5] + vpaddd G, G, [rsp + _DIGEST_SAVE + 64*6] + vpaddd H, H, [rsp + _DIGEST_SAVE + 64*7] + + jmp lloop + +lastLoop: + ; Process last 16 rounds +%assign I 48 +%assign J 0 +%rep 16 + PROCESS_LOOP APPEND(W,J), I +%assign I (I+1) +%assign J (J+1) +%endrep + + ; Add old digest + vpaddd A, A, [rsp + _DIGEST_SAVE + 64*0] + vpaddd B, B, [rsp + _DIGEST_SAVE + 64*1] + vpaddd C, C, [rsp + _DIGEST_SAVE + 64*2] + vpaddd D, D, [rsp + _DIGEST_SAVE + 64*3] + vpaddd E, E, [rsp + _DIGEST_SAVE + 64*4] + vpaddd F, F, [rsp + _DIGEST_SAVE + 64*5] + vpaddd G, G, [rsp + _DIGEST_SAVE + 64*6] + vpaddd H, H, [rsp + _DIGEST_SAVE + 64*7] + + ; Write out digest + ; Do we need to untranspose digests??? + vmovdqu32 [STATE + 0*SHA256_DIGEST_ROW_SIZE], A + vmovdqu32 [STATE + 1*SHA256_DIGEST_ROW_SIZE], B + vmovdqu32 [STATE + 2*SHA256_DIGEST_ROW_SIZE], C + vmovdqu32 [STATE + 3*SHA256_DIGEST_ROW_SIZE], D + vmovdqu32 [STATE + 4*SHA256_DIGEST_ROW_SIZE], E + vmovdqu32 [STATE + 5*SHA256_DIGEST_ROW_SIZE], F + vmovdqu32 [STATE + 6*SHA256_DIGEST_ROW_SIZE], G + vmovdqu32 [STATE + 7*SHA256_DIGEST_ROW_SIZE], H + + ; update input pointers +%assign I 0 +%rep 16 + add [STATE + _data_ptr_sha256 + I*PTR_SZ], IDX +%assign I (I+1) +%endrep + +%ifdef SAFE_DATA + ;; Clear stack frame (8*64 bytes) + vpxorq zmm0, zmm0 +%assign i 0 +%rep 8 + vmovdqa64 [rsp + i*64], zmm0 +%assign i (i+1) +%endrep +%endif + + mov rsp, [rsp + _rsp] + ret + +%ifdef LINUX +section .note.GNU-stack noalloc noexec nowrite progbits +%endif diff --git a/src/spdk/intel-ipsec-mb/avx512/sha512_x8_avx512.asm b/src/spdk/intel-ipsec-mb/avx512/sha512_x8_avx512.asm new file mode 100644 index 000000000..48532c3fb --- /dev/null +++ b/src/spdk/intel-ipsec-mb/avx512/sha512_x8_avx512.asm @@ -0,0 +1,595 @@ +;; +;; Copyright (c) 2017-2018, Intel Corporation +;; +;; Redistribution and use in source and binary forms, with or without +;; modification, are permitted provided that the following conditions are met: +;; +;; * Redistributions of source code must retain the above copyright notice, +;; this list of conditions and the following disclaimer. +;; * Redistributions in binary form must reproduce the above copyright +;; notice, this list of conditions and the following disclaimer in the +;; documentation and/or other materials provided with the distribution. +;; * Neither the name of Intel Corporation nor the names of its contributors +;; may be used to endorse or promote products derived from this software +;; without specific prior written permission. +;; +;; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +;; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +;; IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +;; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE +;; FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +;; DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +;; SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +;; CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +;; OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +;; + +;; Stack must be aligned to 32 bytes before call +;; +;; Registers: RAX RBX RCX RDX RBP RSI RDI R8 R9 R10 R11 R12 R13 R14 R15 +;; ----------------------------------------------------------- +;; Windows clobbers: RAX RDX RDI R8 R9 R10 R11 R12 R13 R14 R15 +;; Windows preserves: RBX RCX RBP RSI +;; ----------------------------------------------------------- +;; Linux clobbers: RAX RDX RSI R8 R9 R10 R11 R12 R13 R14 R15 +;; Linux preserves: RBX RCX RBP RDI +;; ----------------------------------------------------------- +;; Clobbers ZMM0-31 + +;; code to compute quad SHA512 using AVX512 + +%include "include/os.asm" +;%define DO_DBGPRINT +%include "include/dbgprint.asm" +%include "mb_mgr_datastruct.asm" +%include "include/transpose_avx512.asm" + +%define APPEND(a,b) a %+ b + +%ifdef LINUX +; Linux register definitions +%define arg1 rdi +%define arg2 rsi +%define arg3 rcx +%define arg4 rdx +%else +; Windows definitions +%define arg1 rcx +%define arg2 rdx +%define arg3 rsi +%define arg4 rdi +%endif + +%define STATE arg1 +%define INP_SIZE arg2 + +%define IDX arg4 +%define TBL r8 + +;; retaining XMM_SAVE, because the top half of YMM registers no saving required, only bottom half, the XMM part +%define NUM_LANES 8 +%define XMM_SAVE (15-5)*16 +%define SZ 8 +%define SZ8 8 * SZ +%define DIGEST_SZ 8 * SZ8 +%define DIGEST_SAVE NUM_LANES * DIGEST_SZ +%define RSP_SAVE 1*8 + + +; Define Stack Layout +START_FIELDS +;;; name size align +FIELD _DIGEST_SAVE, NUM_LANES*8*64, 64 +FIELD _XMM_SAVE, XMM_SAVE, 16 +FIELD _RSP, 8, 8 +%assign STACK_SPACE _FIELD_OFFSET + +%define inp0 r9 +%define inp1 r10 +%define inp2 r11 +%define inp3 r12 +%define inp4 r13 +%define inp5 r14 +%define inp6 r15 +%define inp7 rax + +%define A zmm0 +%define B zmm1 +%define C zmm2 +%define D zmm3 +%define E zmm4 +%define F zmm5 +%define G zmm6 +%define H zmm7 +%define T1 zmm8 +%define TMP0 zmm9 +%define TMP1 zmm10 +%define TMP2 zmm11 +%define TMP3 zmm12 +%define TMP4 zmm13 +%define TMP5 zmm14 +%define TMP6 zmm15 + + +%define W0 zmm16 +%define W1 zmm17 +%define W2 zmm18 +%define W3 zmm19 +%define W4 zmm20 +%define W5 zmm21 +%define W6 zmm22 +%define W7 zmm23 +%define W8 zmm24 +%define W9 zmm25 +%define W10 zmm26 +%define W11 zmm27 +%define W12 zmm28 +%define W13 zmm29 +%define W14 zmm30 +%define W15 zmm31 + +; from sha256_fips180-2.pdf +; define rotates for Sigma function for main loop steps +%define BIG_SIGMA_0_0 28 ; Sigma0 +%define BIG_SIGMA_0_1 34 +%define BIG_SIGMA_0_2 39 +%define BIG_SIGMA_1_0 14 ; Sigma1 +%define BIG_SIGMA_1_1 18 +%define BIG_SIGMA_1_2 41 + +; define rotates for Sigma function for scheduling steps +%define SMALL_SIGMA_0_0 1 ; sigma0 +%define SMALL_SIGMA_0_1 8 +%define SMALL_SIGMA_0_2 7 +%define SMALL_SIGMA_1_0 19 ; sigma1 +%define SMALL_SIGMA_1_1 61 +%define SMALL_SIGMA_1_2 6 + +%define SHA_MAX_ROUNDS 80 +%define SHA_ROUNDS_LESS_16 (SHA_MAX_ROUNDS - 16) + +%macro ROTATE_ARGS 0 +%xdefine TMP_ H +%xdefine H G +%xdefine G F +%xdefine F E +%xdefine E D +%xdefine D C +%xdefine C B +%xdefine B A +%xdefine A TMP_ +%endm + +;; CH(A, B, C) = (A&B) ^ (~A&C) +;; MAJ(E, F, G) = (E&F) ^ (E&G) ^ (F&G) +;; SIGMA0 = ROR_28 ^ ROR_34 ^ ROR_39 +;; SIGMA1 = ROR_14 ^ ROR_18 ^ ROR_41 +;; sigma0 = ROR_1 ^ ROR_8 ^ SHR_7 +;; sigma1 = ROR_19 ^ ROR_61 ^ SHR_6 + +;; Main processing loop per round +;; equivalent to %macro ROUND_00_15 2 +%macro PROCESS_LOOP 2 +%define %%WT %1 +%define %%ROUND %2 + ;; T1 = H + BIG_SIGMA_1(E) + CH(E, F, G) + Kt + Wt + ;; T2 = BIG_SIGMA_0(A) + MAJ(A, B, C) + ;; H=G, G=F, F=E, E=D+T1, D=C, C=B, B=A, A=T1+T2 + + ;; H becomes T2, then add T1 for A + ;; D becomes D + T1 for E + + vpaddq T1, H, TMP3 ; T1 = H + Kt + vmovdqa32 TMP0, E + ;; compute BIG_SIGMA_1(E) + vprorq TMP1, E, BIG_SIGMA_1_0 ; ROR_14(E) + vprorq TMP2, E, BIG_SIGMA_1_1 ; ROR_18(E) + vprorq TMP3, E, BIG_SIGMA_1_2 ; ROR_41(E) + vpternlogq TMP1, TMP2, TMP3, 0x96 ; TMP1 = BIG_SIGMA_1(E) + vpternlogq TMP0, F, G, 0xCA ; TMP0 = CH(E,F,G) + vpaddq T1, T1, %%WT ; T1 = T1 + Wt + vpaddq T1, T1, TMP0 ; T1 = T1 + CH(E,F,G) + vpaddq T1, T1, TMP1 ; T1 = T1 + BIG_SIGMA_1(E) + vpaddq D, D, T1 ; D = D + T1 + vprorq H, A, BIG_SIGMA_0_0 ;ROR_28(A) + vprorq TMP2, A, BIG_SIGMA_0_1 ;ROR_34(A) + vprorq TMP3, A, BIG_SIGMA_0_2 ;ROR_39(A) + vmovdqa32 TMP0, A + vpternlogq TMP0, B, C, 0xE8 ; TMP0 = MAJ(A,B,C) + vpternlogq H, TMP2, TMP3, 0x96 ; H(T2) = BIG_SIGMA_0(A) + vpaddq H, H, TMP0 ; H(T2) = BIG_SIGMA_0(A) + MAJ(A,B,C) + vpaddq H, H, T1 ; H(A) = H(T2) + T1 + vmovdqa32 TMP3, [TBL + ((%%ROUND+1)*64)] ; Next Kt + + ;; Rotate the args A-H (rotation of names associated with regs) + ROTATE_ARGS +%endmacro + +%macro MSG_SCHED_ROUND_16_79 4 +%define %%WT %1 +%define %%WTp1 %2 +%define %%WTp9 %3 +%define %%WTp14 %4 + vprorq TMP4, %%WTp14, SMALL_SIGMA_1_0 ; ROR_19(Wt-2) + vprorq TMP5, %%WTp14, SMALL_SIGMA_1_1 ; ROR_61(Wt-2) + vpsrlq TMP6, %%WTp14, SMALL_SIGMA_1_2 ; SHR_6(Wt-2) + vpternlogq TMP4, TMP5, TMP6, 0x96 ; TMP4 = sigma_1(Wt-2) + + vpaddq %%WT, %%WT, TMP4 ; Wt = Wt-16 + sigma_1(Wt-2) + vpaddq %%WT, %%WT, %%WTp9 ; Wt = Wt-16 + sigma_1(Wt-2) + Wt-7 + + vprorq TMP4, %%WTp1, SMALL_SIGMA_0_0 ; ROR_1(Wt-15) + vprorq TMP5, %%WTp1, SMALL_SIGMA_0_1 ; ROR_8(Wt-15) + vpsrlq TMP6, %%WTp1, SMALL_SIGMA_0_2 ; SHR_7(Wt-15) + vpternlogq TMP4, TMP5, TMP6, 0x96 ; TMP4 = sigma_0(Wt-15) + + vpaddq %%WT, %%WT, TMP4 ; Wt = Wt-16 + sigma_1(Wt-2) + + ; Wt-7 + sigma_0(Wt-15) + +%endmacro + +section .data +default rel + +align 64 +; 80 constants for SHA512 +; replicating for each lane, thus 8*80 +; to aid in SIMD .. space tradeoff for time! +; local to asm file, used nowhere else +TABLE: + dq 0x428a2f98d728ae22, 0x428a2f98d728ae22, 0x428a2f98d728ae22, 0x428a2f98d728ae22 + dq 0x428a2f98d728ae22, 0x428a2f98d728ae22, 0x428a2f98d728ae22, 0x428a2f98d728ae22 + dq 0x7137449123ef65cd, 0x7137449123ef65cd, 0x7137449123ef65cd, 0x7137449123ef65cd + dq 0x7137449123ef65cd, 0x7137449123ef65cd, 0x7137449123ef65cd, 0x7137449123ef65cd + dq 0xb5c0fbcfec4d3b2f, 0xb5c0fbcfec4d3b2f, 0xb5c0fbcfec4d3b2f, 0xb5c0fbcfec4d3b2f + dq 0xb5c0fbcfec4d3b2f, 0xb5c0fbcfec4d3b2f, 0xb5c0fbcfec4d3b2f, 0xb5c0fbcfec4d3b2f + dq 0xe9b5dba58189dbbc, 0xe9b5dba58189dbbc, 0xe9b5dba58189dbbc, 0xe9b5dba58189dbbc + dq 0xe9b5dba58189dbbc, 0xe9b5dba58189dbbc, 0xe9b5dba58189dbbc, 0xe9b5dba58189dbbc + dq 0x3956c25bf348b538, 0x3956c25bf348b538, 0x3956c25bf348b538, 0x3956c25bf348b538 + dq 0x3956c25bf348b538, 0x3956c25bf348b538, 0x3956c25bf348b538, 0x3956c25bf348b538 + dq 0x59f111f1b605d019, 0x59f111f1b605d019, 0x59f111f1b605d019, 0x59f111f1b605d019 + dq 0x59f111f1b605d019, 0x59f111f1b605d019, 0x59f111f1b605d019, 0x59f111f1b605d019 + dq 0x923f82a4af194f9b, 0x923f82a4af194f9b, 0x923f82a4af194f9b, 0x923f82a4af194f9b + dq 0x923f82a4af194f9b, 0x923f82a4af194f9b, 0x923f82a4af194f9b, 0x923f82a4af194f9b + dq 0xab1c5ed5da6d8118, 0xab1c5ed5da6d8118, 0xab1c5ed5da6d8118, 0xab1c5ed5da6d8118 + dq 0xab1c5ed5da6d8118, 0xab1c5ed5da6d8118, 0xab1c5ed5da6d8118, 0xab1c5ed5da6d8118 + dq 0xd807aa98a3030242, 0xd807aa98a3030242, 0xd807aa98a3030242, 0xd807aa98a3030242 + dq 0xd807aa98a3030242, 0xd807aa98a3030242, 0xd807aa98a3030242, 0xd807aa98a3030242 + dq 0x12835b0145706fbe, 0x12835b0145706fbe, 0x12835b0145706fbe, 0x12835b0145706fbe + dq 0x12835b0145706fbe, 0x12835b0145706fbe, 0x12835b0145706fbe, 0x12835b0145706fbe + dq 0x243185be4ee4b28c, 0x243185be4ee4b28c, 0x243185be4ee4b28c, 0x243185be4ee4b28c + dq 0x243185be4ee4b28c, 0x243185be4ee4b28c, 0x243185be4ee4b28c, 0x243185be4ee4b28c + dq 0x550c7dc3d5ffb4e2, 0x550c7dc3d5ffb4e2, 0x550c7dc3d5ffb4e2, 0x550c7dc3d5ffb4e2 + dq 0x550c7dc3d5ffb4e2, 0x550c7dc3d5ffb4e2, 0x550c7dc3d5ffb4e2, 0x550c7dc3d5ffb4e2 + dq 0x72be5d74f27b896f, 0x72be5d74f27b896f, 0x72be5d74f27b896f, 0x72be5d74f27b896f + dq 0x72be5d74f27b896f, 0x72be5d74f27b896f, 0x72be5d74f27b896f, 0x72be5d74f27b896f + dq 0x80deb1fe3b1696b1, 0x80deb1fe3b1696b1, 0x80deb1fe3b1696b1, 0x80deb1fe3b1696b1 + dq 0x80deb1fe3b1696b1, 0x80deb1fe3b1696b1, 0x80deb1fe3b1696b1, 0x80deb1fe3b1696b1 + dq 0x9bdc06a725c71235, 0x9bdc06a725c71235, 0x9bdc06a725c71235, 0x9bdc06a725c71235 + dq 0x9bdc06a725c71235, 0x9bdc06a725c71235, 0x9bdc06a725c71235, 0x9bdc06a725c71235 + dq 0xc19bf174cf692694, 0xc19bf174cf692694, 0xc19bf174cf692694, 0xc19bf174cf692694 + dq 0xc19bf174cf692694, 0xc19bf174cf692694, 0xc19bf174cf692694, 0xc19bf174cf692694 + dq 0xe49b69c19ef14ad2, 0xe49b69c19ef14ad2, 0xe49b69c19ef14ad2, 0xe49b69c19ef14ad2 + dq 0xe49b69c19ef14ad2, 0xe49b69c19ef14ad2, 0xe49b69c19ef14ad2, 0xe49b69c19ef14ad2 + dq 0xefbe4786384f25e3, 0xefbe4786384f25e3, 0xefbe4786384f25e3, 0xefbe4786384f25e3 + dq 0xefbe4786384f25e3, 0xefbe4786384f25e3, 0xefbe4786384f25e3, 0xefbe4786384f25e3 + dq 0x0fc19dc68b8cd5b5, 0x0fc19dc68b8cd5b5, 0x0fc19dc68b8cd5b5, 0x0fc19dc68b8cd5b5 + dq 0x0fc19dc68b8cd5b5, 0x0fc19dc68b8cd5b5, 0x0fc19dc68b8cd5b5, 0x0fc19dc68b8cd5b5 + dq 0x240ca1cc77ac9c65, 0x240ca1cc77ac9c65, 0x240ca1cc77ac9c65, 0x240ca1cc77ac9c65 + dq 0x240ca1cc77ac9c65, 0x240ca1cc77ac9c65, 0x240ca1cc77ac9c65, 0x240ca1cc77ac9c65 + dq 0x2de92c6f592b0275, 0x2de92c6f592b0275, 0x2de92c6f592b0275, 0x2de92c6f592b0275 + dq 0x2de92c6f592b0275, 0x2de92c6f592b0275, 0x2de92c6f592b0275, 0x2de92c6f592b0275 + dq 0x4a7484aa6ea6e483, 0x4a7484aa6ea6e483, 0x4a7484aa6ea6e483, 0x4a7484aa6ea6e483 + dq 0x4a7484aa6ea6e483, 0x4a7484aa6ea6e483, 0x4a7484aa6ea6e483, 0x4a7484aa6ea6e483 + dq 0x5cb0a9dcbd41fbd4, 0x5cb0a9dcbd41fbd4, 0x5cb0a9dcbd41fbd4, 0x5cb0a9dcbd41fbd4 + dq 0x5cb0a9dcbd41fbd4, 0x5cb0a9dcbd41fbd4, 0x5cb0a9dcbd41fbd4, 0x5cb0a9dcbd41fbd4 + dq 0x76f988da831153b5, 0x76f988da831153b5, 0x76f988da831153b5, 0x76f988da831153b5 + dq 0x76f988da831153b5, 0x76f988da831153b5, 0x76f988da831153b5, 0x76f988da831153b5 + dq 0x983e5152ee66dfab, 0x983e5152ee66dfab, 0x983e5152ee66dfab, 0x983e5152ee66dfab + dq 0x983e5152ee66dfab, 0x983e5152ee66dfab, 0x983e5152ee66dfab, 0x983e5152ee66dfab + dq 0xa831c66d2db43210, 0xa831c66d2db43210, 0xa831c66d2db43210, 0xa831c66d2db43210 + dq 0xa831c66d2db43210, 0xa831c66d2db43210, 0xa831c66d2db43210, 0xa831c66d2db43210 + dq 0xb00327c898fb213f, 0xb00327c898fb213f, 0xb00327c898fb213f, 0xb00327c898fb213f + dq 0xb00327c898fb213f, 0xb00327c898fb213f, 0xb00327c898fb213f, 0xb00327c898fb213f + dq 0xbf597fc7beef0ee4, 0xbf597fc7beef0ee4, 0xbf597fc7beef0ee4, 0xbf597fc7beef0ee4 + dq 0xbf597fc7beef0ee4, 0xbf597fc7beef0ee4, 0xbf597fc7beef0ee4, 0xbf597fc7beef0ee4 + dq 0xc6e00bf33da88fc2, 0xc6e00bf33da88fc2, 0xc6e00bf33da88fc2, 0xc6e00bf33da88fc2 + dq 0xc6e00bf33da88fc2, 0xc6e00bf33da88fc2, 0xc6e00bf33da88fc2, 0xc6e00bf33da88fc2 + dq 0xd5a79147930aa725, 0xd5a79147930aa725, 0xd5a79147930aa725, 0xd5a79147930aa725 + dq 0xd5a79147930aa725, 0xd5a79147930aa725, 0xd5a79147930aa725, 0xd5a79147930aa725 + dq 0x06ca6351e003826f, 0x06ca6351e003826f, 0x06ca6351e003826f, 0x06ca6351e003826f + dq 0x06ca6351e003826f, 0x06ca6351e003826f, 0x06ca6351e003826f, 0x06ca6351e003826f + dq 0x142929670a0e6e70, 0x142929670a0e6e70, 0x142929670a0e6e70, 0x142929670a0e6e70 + dq 0x142929670a0e6e70, 0x142929670a0e6e70, 0x142929670a0e6e70, 0x142929670a0e6e70 + dq 0x27b70a8546d22ffc, 0x27b70a8546d22ffc, 0x27b70a8546d22ffc, 0x27b70a8546d22ffc + dq 0x27b70a8546d22ffc, 0x27b70a8546d22ffc, 0x27b70a8546d22ffc, 0x27b70a8546d22ffc + dq 0x2e1b21385c26c926, 0x2e1b21385c26c926, 0x2e1b21385c26c926, 0x2e1b21385c26c926 + dq 0x2e1b21385c26c926, 0x2e1b21385c26c926, 0x2e1b21385c26c926, 0x2e1b21385c26c926 + dq 0x4d2c6dfc5ac42aed, 0x4d2c6dfc5ac42aed, 0x4d2c6dfc5ac42aed, 0x4d2c6dfc5ac42aed + dq 0x4d2c6dfc5ac42aed, 0x4d2c6dfc5ac42aed, 0x4d2c6dfc5ac42aed, 0x4d2c6dfc5ac42aed + dq 0x53380d139d95b3df, 0x53380d139d95b3df, 0x53380d139d95b3df, 0x53380d139d95b3df + dq 0x53380d139d95b3df, 0x53380d139d95b3df, 0x53380d139d95b3df, 0x53380d139d95b3df + dq 0x650a73548baf63de, 0x650a73548baf63de, 0x650a73548baf63de, 0x650a73548baf63de + dq 0x650a73548baf63de, 0x650a73548baf63de, 0x650a73548baf63de, 0x650a73548baf63de + dq 0x766a0abb3c77b2a8, 0x766a0abb3c77b2a8, 0x766a0abb3c77b2a8, 0x766a0abb3c77b2a8 + dq 0x766a0abb3c77b2a8, 0x766a0abb3c77b2a8, 0x766a0abb3c77b2a8, 0x766a0abb3c77b2a8 + dq 0x81c2c92e47edaee6, 0x81c2c92e47edaee6, 0x81c2c92e47edaee6, 0x81c2c92e47edaee6 + dq 0x81c2c92e47edaee6, 0x81c2c92e47edaee6, 0x81c2c92e47edaee6, 0x81c2c92e47edaee6 + dq 0x92722c851482353b, 0x92722c851482353b, 0x92722c851482353b, 0x92722c851482353b + dq 0x92722c851482353b, 0x92722c851482353b, 0x92722c851482353b, 0x92722c851482353b + dq 0xa2bfe8a14cf10364, 0xa2bfe8a14cf10364, 0xa2bfe8a14cf10364, 0xa2bfe8a14cf10364 + dq 0xa2bfe8a14cf10364, 0xa2bfe8a14cf10364, 0xa2bfe8a14cf10364, 0xa2bfe8a14cf10364 + dq 0xa81a664bbc423001, 0xa81a664bbc423001, 0xa81a664bbc423001, 0xa81a664bbc423001 + dq 0xa81a664bbc423001, 0xa81a664bbc423001, 0xa81a664bbc423001, 0xa81a664bbc423001 + dq 0xc24b8b70d0f89791, 0xc24b8b70d0f89791, 0xc24b8b70d0f89791, 0xc24b8b70d0f89791 + dq 0xc24b8b70d0f89791, 0xc24b8b70d0f89791, 0xc24b8b70d0f89791, 0xc24b8b70d0f89791 + dq 0xc76c51a30654be30, 0xc76c51a30654be30, 0xc76c51a30654be30, 0xc76c51a30654be30 + dq 0xc76c51a30654be30, 0xc76c51a30654be30, 0xc76c51a30654be30, 0xc76c51a30654be30 + dq 0xd192e819d6ef5218, 0xd192e819d6ef5218, 0xd192e819d6ef5218, 0xd192e819d6ef5218 + dq 0xd192e819d6ef5218, 0xd192e819d6ef5218, 0xd192e819d6ef5218, 0xd192e819d6ef5218 + dq 0xd69906245565a910, 0xd69906245565a910, 0xd69906245565a910, 0xd69906245565a910 + dq 0xd69906245565a910, 0xd69906245565a910, 0xd69906245565a910, 0xd69906245565a910 + dq 0xf40e35855771202a, 0xf40e35855771202a, 0xf40e35855771202a, 0xf40e35855771202a + dq 0xf40e35855771202a, 0xf40e35855771202a, 0xf40e35855771202a, 0xf40e35855771202a + dq 0x106aa07032bbd1b8, 0x106aa07032bbd1b8, 0x106aa07032bbd1b8, 0x106aa07032bbd1b8 + dq 0x106aa07032bbd1b8, 0x106aa07032bbd1b8, 0x106aa07032bbd1b8, 0x106aa07032bbd1b8 + dq 0x19a4c116b8d2d0c8, 0x19a4c116b8d2d0c8, 0x19a4c116b8d2d0c8, 0x19a4c116b8d2d0c8 + dq 0x19a4c116b8d2d0c8, 0x19a4c116b8d2d0c8, 0x19a4c116b8d2d0c8, 0x19a4c116b8d2d0c8 + dq 0x1e376c085141ab53, 0x1e376c085141ab53, 0x1e376c085141ab53, 0x1e376c085141ab53 + dq 0x1e376c085141ab53, 0x1e376c085141ab53, 0x1e376c085141ab53, 0x1e376c085141ab53 + dq 0x2748774cdf8eeb99, 0x2748774cdf8eeb99, 0x2748774cdf8eeb99, 0x2748774cdf8eeb99 + dq 0x2748774cdf8eeb99, 0x2748774cdf8eeb99, 0x2748774cdf8eeb99, 0x2748774cdf8eeb99 + dq 0x34b0bcb5e19b48a8, 0x34b0bcb5e19b48a8, 0x34b0bcb5e19b48a8, 0x34b0bcb5e19b48a8 + dq 0x34b0bcb5e19b48a8, 0x34b0bcb5e19b48a8, 0x34b0bcb5e19b48a8, 0x34b0bcb5e19b48a8 + dq 0x391c0cb3c5c95a63, 0x391c0cb3c5c95a63, 0x391c0cb3c5c95a63, 0x391c0cb3c5c95a63 + dq 0x391c0cb3c5c95a63, 0x391c0cb3c5c95a63, 0x391c0cb3c5c95a63, 0x391c0cb3c5c95a63 + dq 0x4ed8aa4ae3418acb, 0x4ed8aa4ae3418acb, 0x4ed8aa4ae3418acb, 0x4ed8aa4ae3418acb + dq 0x4ed8aa4ae3418acb, 0x4ed8aa4ae3418acb, 0x4ed8aa4ae3418acb, 0x4ed8aa4ae3418acb + dq 0x5b9cca4f7763e373, 0x5b9cca4f7763e373, 0x5b9cca4f7763e373, 0x5b9cca4f7763e373 + dq 0x5b9cca4f7763e373, 0x5b9cca4f7763e373, 0x5b9cca4f7763e373, 0x5b9cca4f7763e373 + dq 0x682e6ff3d6b2b8a3, 0x682e6ff3d6b2b8a3, 0x682e6ff3d6b2b8a3, 0x682e6ff3d6b2b8a3 + dq 0x682e6ff3d6b2b8a3, 0x682e6ff3d6b2b8a3, 0x682e6ff3d6b2b8a3, 0x682e6ff3d6b2b8a3 + dq 0x748f82ee5defb2fc, 0x748f82ee5defb2fc, 0x748f82ee5defb2fc, 0x748f82ee5defb2fc + dq 0x748f82ee5defb2fc, 0x748f82ee5defb2fc, 0x748f82ee5defb2fc, 0x748f82ee5defb2fc + dq 0x78a5636f43172f60, 0x78a5636f43172f60, 0x78a5636f43172f60, 0x78a5636f43172f60 + dq 0x78a5636f43172f60, 0x78a5636f43172f60, 0x78a5636f43172f60, 0x78a5636f43172f60 + dq 0x84c87814a1f0ab72, 0x84c87814a1f0ab72, 0x84c87814a1f0ab72, 0x84c87814a1f0ab72 + dq 0x84c87814a1f0ab72, 0x84c87814a1f0ab72, 0x84c87814a1f0ab72, 0x84c87814a1f0ab72 + dq 0x8cc702081a6439ec, 0x8cc702081a6439ec, 0x8cc702081a6439ec, 0x8cc702081a6439ec + dq 0x8cc702081a6439ec, 0x8cc702081a6439ec, 0x8cc702081a6439ec, 0x8cc702081a6439ec + dq 0x90befffa23631e28, 0x90befffa23631e28, 0x90befffa23631e28, 0x90befffa23631e28 + dq 0x90befffa23631e28, 0x90befffa23631e28, 0x90befffa23631e28, 0x90befffa23631e28 + dq 0xa4506cebde82bde9, 0xa4506cebde82bde9, 0xa4506cebde82bde9, 0xa4506cebde82bde9 + dq 0xa4506cebde82bde9, 0xa4506cebde82bde9, 0xa4506cebde82bde9, 0xa4506cebde82bde9 + dq 0xbef9a3f7b2c67915, 0xbef9a3f7b2c67915, 0xbef9a3f7b2c67915, 0xbef9a3f7b2c67915 + dq 0xbef9a3f7b2c67915, 0xbef9a3f7b2c67915, 0xbef9a3f7b2c67915, 0xbef9a3f7b2c67915 + dq 0xc67178f2e372532b, 0xc67178f2e372532b, 0xc67178f2e372532b, 0xc67178f2e372532b + dq 0xc67178f2e372532b, 0xc67178f2e372532b, 0xc67178f2e372532b, 0xc67178f2e372532b + dq 0xca273eceea26619c, 0xca273eceea26619c, 0xca273eceea26619c, 0xca273eceea26619c + dq 0xca273eceea26619c, 0xca273eceea26619c, 0xca273eceea26619c, 0xca273eceea26619c + dq 0xd186b8c721c0c207, 0xd186b8c721c0c207, 0xd186b8c721c0c207, 0xd186b8c721c0c207 + dq 0xd186b8c721c0c207, 0xd186b8c721c0c207, 0xd186b8c721c0c207, 0xd186b8c721c0c207 + dq 0xeada7dd6cde0eb1e, 0xeada7dd6cde0eb1e, 0xeada7dd6cde0eb1e, 0xeada7dd6cde0eb1e + dq 0xeada7dd6cde0eb1e, 0xeada7dd6cde0eb1e, 0xeada7dd6cde0eb1e, 0xeada7dd6cde0eb1e + dq 0xf57d4f7fee6ed178, 0xf57d4f7fee6ed178, 0xf57d4f7fee6ed178, 0xf57d4f7fee6ed178 + dq 0xf57d4f7fee6ed178, 0xf57d4f7fee6ed178, 0xf57d4f7fee6ed178, 0xf57d4f7fee6ed178 + dq 0x06f067aa72176fba, 0x06f067aa72176fba, 0x06f067aa72176fba, 0x06f067aa72176fba + dq 0x06f067aa72176fba, 0x06f067aa72176fba, 0x06f067aa72176fba, 0x06f067aa72176fba + dq 0x0a637dc5a2c898a6, 0x0a637dc5a2c898a6, 0x0a637dc5a2c898a6, 0x0a637dc5a2c898a6 + dq 0x0a637dc5a2c898a6, 0x0a637dc5a2c898a6, 0x0a637dc5a2c898a6, 0x0a637dc5a2c898a6 + dq 0x113f9804bef90dae, 0x113f9804bef90dae, 0x113f9804bef90dae, 0x113f9804bef90dae + dq 0x113f9804bef90dae, 0x113f9804bef90dae, 0x113f9804bef90dae, 0x113f9804bef90dae + dq 0x1b710b35131c471b, 0x1b710b35131c471b, 0x1b710b35131c471b, 0x1b710b35131c471b + dq 0x1b710b35131c471b, 0x1b710b35131c471b, 0x1b710b35131c471b, 0x1b710b35131c471b + dq 0x28db77f523047d84, 0x28db77f523047d84, 0x28db77f523047d84, 0x28db77f523047d84 + dq 0x28db77f523047d84, 0x28db77f523047d84, 0x28db77f523047d84, 0x28db77f523047d84 + dq 0x32caab7b40c72493, 0x32caab7b40c72493, 0x32caab7b40c72493, 0x32caab7b40c72493 + dq 0x32caab7b40c72493, 0x32caab7b40c72493, 0x32caab7b40c72493, 0x32caab7b40c72493 + dq 0x3c9ebe0a15c9bebc, 0x3c9ebe0a15c9bebc, 0x3c9ebe0a15c9bebc, 0x3c9ebe0a15c9bebc + dq 0x3c9ebe0a15c9bebc, 0x3c9ebe0a15c9bebc, 0x3c9ebe0a15c9bebc, 0x3c9ebe0a15c9bebc + dq 0x431d67c49c100d4c, 0x431d67c49c100d4c, 0x431d67c49c100d4c, 0x431d67c49c100d4c + dq 0x431d67c49c100d4c, 0x431d67c49c100d4c, 0x431d67c49c100d4c, 0x431d67c49c100d4c + dq 0x4cc5d4becb3e42b6, 0x4cc5d4becb3e42b6, 0x4cc5d4becb3e42b6, 0x4cc5d4becb3e42b6 + dq 0x4cc5d4becb3e42b6, 0x4cc5d4becb3e42b6, 0x4cc5d4becb3e42b6, 0x4cc5d4becb3e42b6 + dq 0x597f299cfc657e2a, 0x597f299cfc657e2a, 0x597f299cfc657e2a, 0x597f299cfc657e2a + dq 0x597f299cfc657e2a, 0x597f299cfc657e2a, 0x597f299cfc657e2a, 0x597f299cfc657e2a + dq 0x5fcb6fab3ad6faec, 0x5fcb6fab3ad6faec, 0x5fcb6fab3ad6faec, 0x5fcb6fab3ad6faec + dq 0x5fcb6fab3ad6faec, 0x5fcb6fab3ad6faec, 0x5fcb6fab3ad6faec, 0x5fcb6fab3ad6faec + dq 0x6c44198c4a475817, 0x6c44198c4a475817, 0x6c44198c4a475817, 0x6c44198c4a475817 + dq 0x6c44198c4a475817, 0x6c44198c4a475817, 0x6c44198c4a475817, 0x6c44198c4a475817 + +align 64 +; this does the big endian to little endian conversion over a quad word .. ZMM +;; shuffle on ZMM is shuffle on 4 XMM size chunks, 128 bits +PSHUFFLE_BYTE_FLIP_MASK: + ;ddq 0x08090a0b0c0d0e0f0001020304050607 + dq 0x0001020304050607, 0x08090a0b0c0d0e0f + ;ddq 0x18191a1b1c1d1e1f1011121314151617 + dq 0x1011121314151617, 0x18191a1b1c1d1e1f + ;ddq 0x28292a2b2c2d2e2f2021222324252627 + dq 0x2021222324252627, 0x28292a2b2c2d2e2f + ;ddq 0x38393a3b3c3d3e3f3031323334353637 + dq 0x3031323334353637, 0x38393a3b3c3d3e3f + +section .text + +;; void sha512_x8_avx512(void *input_data, UINT64 *digest[NUM_LANES], const int size) +;; arg 1 : rcx : pointer to input data +;; arg 2 : rdx : pointer to UINT64 digest[8][num_lanes] +;; arg 3 : size in message block lengths (= 128 bytes) +MKGLOBAL(sha512_x8_avx512,function,internal) +align 64 +sha512_x8_avx512: + mov rax, rsp + sub rsp, STACK_SPACE + and rsp, ~63 ; align stack to multiple of 64 + mov [rsp + _RSP], rax + + ;; Initialize digests ; organized uint64 digest[8][num_lanes]; no transpose required + ;; Digest is an array of pointers to digests + vmovdqu32 A, [STATE + 0*SHA512_DIGEST_ROW_SIZE] + vmovdqu32 B, [STATE + 1*SHA512_DIGEST_ROW_SIZE] + vmovdqu32 C, [STATE + 2*SHA512_DIGEST_ROW_SIZE] + vmovdqu32 D, [STATE + 3*SHA512_DIGEST_ROW_SIZE] + vmovdqu32 E, [STATE + 4*SHA512_DIGEST_ROW_SIZE] + vmovdqu32 F, [STATE + 5*SHA512_DIGEST_ROW_SIZE] + vmovdqu32 G, [STATE + 6*SHA512_DIGEST_ROW_SIZE] + vmovdqu32 H, [STATE + 7*SHA512_DIGEST_ROW_SIZE] + + lea TBL,[rel TABLE] + xor IDX, IDX + ;; Read in input data address, saving them in registers because + ;; they will serve as variables, which we shall keep incrementing + mov inp0, [STATE + _data_ptr_sha512 + 0*PTR_SZ] + mov inp1, [STATE + _data_ptr_sha512 + 1*PTR_SZ] + mov inp2, [STATE + _data_ptr_sha512 + 2*PTR_SZ] + mov inp3, [STATE + _data_ptr_sha512 + 3*PTR_SZ] + mov inp4, [STATE + _data_ptr_sha512 + 4*PTR_SZ] + mov inp5, [STATE + _data_ptr_sha512 + 5*PTR_SZ] + mov inp6, [STATE + _data_ptr_sha512 + 6*PTR_SZ] + mov inp7, [STATE + _data_ptr_sha512 + 7*PTR_SZ] + jmp lloop + +align 32 +lloop: + ;; Load 64-byte blocks of data into ZMM registers before + ;; performing a 8x8 64-bit transpose. + ;; To speed up the transpose, data is loaded in chunks of 32 bytes, + ;; interleaving data between lane X and lane X+4. + ;; This way, final shuffles between top half and bottom half + ;; of the matrix are avoided. + TRANSPOSE8_U64_LOAD8 W0, W1, W2, W3, W4, W5, W6, W7, \ + inp0, inp1, inp2, inp3, inp4, inp5, \ + inp6, inp7, IDX + + TRANSPOSE8_U64 W0, W1, W2, W3, W4, W5, W6, W7, TMP0, TMP1, TMP2, TMP3 + ;; Load next 512 bytes + TRANSPOSE8_U64_LOAD8 W8, W9, W10, W11, W12, W13, W14, W15, \ + inp0, inp1, inp2, inp3, inp4, inp5, \ + inp6, inp7, IDX+SZ8 + + TRANSPOSE8_U64 W8, W9, W10, W11, W12, W13, W14, W15, TMP0, TMP1, TMP2, TMP3 + + vmovdqa32 TMP2, [rel PSHUFFLE_BYTE_FLIP_MASK] + + vmovdqa32 TMP3, [TBL] ; First K + + ; Save digests for later addition + vmovdqa32 [rsp + _DIGEST_SAVE + 64*0], A + vmovdqa32 [rsp + _DIGEST_SAVE + 64*1], B + vmovdqa32 [rsp + _DIGEST_SAVE + 64*2], C + vmovdqa32 [rsp + _DIGEST_SAVE + 64*3], D + vmovdqa32 [rsp + _DIGEST_SAVE + 64*4], E + vmovdqa32 [rsp + _DIGEST_SAVE + 64*5], F + vmovdqa32 [rsp + _DIGEST_SAVE + 64*6], G + vmovdqa32 [rsp + _DIGEST_SAVE + 64*7], H + + add IDX, 128 ; increment by message block length in bytes + +%assign I 0 +%rep 16 +;;; little endian to big endian + vpshufb APPEND(W,I), APPEND(W,I), TMP2 +%assign I (I+1) +%endrep + + ; MSG Schedule for W0-W15 is now complete in registers + ; Process first (max-rounds -16) + ; Calculate next Wt+16 after processing is complete and Wt is unneeded + ; PROCESS_LOOP_00_79 APPEND(W,J), I, APPEND(W,K), APPEND(W,L), APPEND(W,M) + +%assign I 0 +%assign J 0 +%assign K 1 +%assign L 9 +%assign M 14 +%rep SHA_ROUNDS_LESS_16 + PROCESS_LOOP APPEND(W,J), I + MSG_SCHED_ROUND_16_79 APPEND(W,J), APPEND(W,K), APPEND(W,L), APPEND(W,M) +%assign I (I+1) +%assign J ((J+1)% 16) +%assign K ((K+1)% 16) +%assign L ((L+1)% 16) +%assign M ((M+1)% 16) +%endrep + ; Check is this is the last block + sub INP_SIZE, 1 + je lastLoop + + ; Process last 16 rounds + ; Read in next block msg data for use in first 16 words of msg sched +%assign I SHA_ROUNDS_LESS_16 +%assign J 0 +%rep 16 + PROCESS_LOOP APPEND(W,J), I +%assign I (I+1) +%assign J (J+1) +%endrep + ; Add old digest + vpaddq A, A, [rsp + _DIGEST_SAVE + 64*0] + vpaddq B, B, [rsp + _DIGEST_SAVE + 64*1] + vpaddq C, C, [rsp + _DIGEST_SAVE + 64*2] + vpaddq D, D, [rsp + _DIGEST_SAVE + 64*3] + vpaddq E, E, [rsp + _DIGEST_SAVE + 64*4] + vpaddq F, F, [rsp + _DIGEST_SAVE + 64*5] + vpaddq G, G, [rsp + _DIGEST_SAVE + 64*6] + vpaddq H, H, [rsp + _DIGEST_SAVE + 64*7] + + jmp lloop + +align 32 +lastLoop: + ; Process last 16 rounds +%assign I SHA_ROUNDS_LESS_16 +%assign J 0 +%rep 16 + PROCESS_LOOP APPEND(W,J), I +%assign I (I+1) +%assign J (J+1) +%endrep + + ; Add old digest + vpaddq A, A, [rsp + _DIGEST_SAVE + 64*0] + vpaddq B, B, [rsp + _DIGEST_SAVE + 64*1] + vpaddq C, C, [rsp + _DIGEST_SAVE + 64*2] + vpaddq D, D, [rsp + _DIGEST_SAVE + 64*3] + vpaddq E, E, [rsp + _DIGEST_SAVE + 64*4] + vpaddq F, F, [rsp + _DIGEST_SAVE + 64*5] + vpaddq G, G, [rsp + _DIGEST_SAVE + 64*6] + vpaddq H, H, [rsp + _DIGEST_SAVE + 64*7] + + ; Write out digest + ;; results in A, B, C, D, E, F, G, H + vmovdqu32 [STATE + 0*SHA512_DIGEST_ROW_SIZE], A + vmovdqu32 [STATE + 1*SHA512_DIGEST_ROW_SIZE], B + vmovdqu32 [STATE + 2*SHA512_DIGEST_ROW_SIZE], C + vmovdqu32 [STATE + 3*SHA512_DIGEST_ROW_SIZE], D + vmovdqu32 [STATE + 4*SHA512_DIGEST_ROW_SIZE], E + vmovdqu32 [STATE + 5*SHA512_DIGEST_ROW_SIZE], F + vmovdqu32 [STATE + 6*SHA512_DIGEST_ROW_SIZE], G + vmovdqu32 [STATE + 7*SHA512_DIGEST_ROW_SIZE], H + + ; update input pointers +%assign I 0 +%rep 8 + add [STATE + _data_ptr_sha512 + I*PTR_SZ], IDX +%assign I (I+1) +%endrep + + +%ifdef SAFE_DATA + ;; Clear stack frame ((NUM_LANES*8)*64 bytes) + vpxorq zmm0, zmm0 +%assign i 0 +%rep (NUM_LANES*8) + vmovdqa64 [rsp + i*64], zmm0 +%assign i (i+1) +%endrep +%endif + mov rsp, [rsp + _RSP] +;hash_done: + ret + +%ifdef LINUX +section .note.GNU-stack noalloc noexec nowrite progbits +%endif |