path: root/src/spdk/intel-ipsec-mb/avx512/des_x16_avx512.asm

;;
;; Copyright (c) 2017-2019, Intel Corporation
;;
;; Redistribution and use in source and binary forms, with or without
;; modification, are permitted provided that the following conditions are met:
;;
;;     * Redistributions of source code must retain the above copyright notice,
;;       this list of conditions and the following disclaimer.
;;     * Redistributions in binary form must reproduce the above copyright
;;       notice, this list of conditions and the following disclaimer in the
;;       documentation and/or other materials provided with the distribution.
;;     * Neither the name of Intel Corporation nor the names of its contributors
;;       may be used to endorse or promote products derived from this software
;;       without specific prior written permission.
;;
;; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
;; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
;; IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
;; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
;; FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
;; DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
;; SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
;; CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
;; OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
;;

;; Authors:
;;   Shay Gueron (1, 2), Regev Shemy (2), Tomasz kantecki (2)
;;   (1) University of Haifa, Israel
;;   (2) Intel Corporation

;; In System V AMD64 ABI
;;	calle saves: RBX, RBP, R12-R15
;; Windows x64 ABI
;;	calle saves: RBX, RBP, RDI, RSI, RSP, R12-R15

;;
;; Registers:		RAX RBX RCX RDX RBP RSI RDI R8  R9  R10 R11 R12 R13 R14 R15
;;			-----------------------------------------------------------
;; Windows clobbers:	RAX                         R8  R9  R10 R11
;; Windows preserves:	    RBX RCX RDX RBP RSI RDI                 R12 R13 R14 R15
;;			-----------------------------------------------------------
;; Linux clobbers:	RAX     RCX RDX                     R10 R11
;; Linux preserves:	    RBX         RBP RSI RDI R8  R9          R12 R13 R14 R15
;;			-----------------------------------------------------------
;; Clobbers ZMM0-31 and K1 to K7

%include "include/os.asm"
%include "include/reg_sizes.asm"
%include "mb_mgr_datastruct.asm"
%include "constants.asm"
;%define DO_DBGPRINT
%include "include/dbgprint.asm"

%ifdef LINUX
%define arg1	rdi
%define arg2	rsi
%define arg3	rdx
%define arg4	rcx
%else
%define arg1	rcx
%define arg2	rdx
%define arg3	r8
%define arg4	r9
%endif

%define STATE	arg1
%define SIZE	arg2

%define OFFSET  rax

%define IA0     arg3
%define IA1     arg4
%define IA2     r10

%define INP0	r11
%define INP1	r12
%define INP2	r13
%define INP3	r14
%define INP4	r15

%define KSOFFSET r11

%define ZW0     zmm0
%define ZW1     zmm1
%define ZW2     zmm2
%define ZW3     zmm3
%define ZW4     zmm4
%define ZW5     zmm5
%define ZW6     zmm6
%define ZW7     zmm7
%define ZW8     zmm8
%define ZW9     zmm9
%define ZW10    zmm10
%define ZW11    zmm11
%define ZW12    zmm12
%define ZW13    zmm13
%define ZW14    zmm14
%define ZW15    zmm15

%define ZIV0    zmm16
%define ZIV1    zmm17

%define ZTMP0   zmm18
%define ZTMP1   zmm19
%define ZTMP2   zmm20
%define ZTMP3   zmm21
%define ZTMP4   zmm22
%define ZTMP5   zmm23
%define ZTMP6   zmm24
%define ZTMP7   zmm25
%define ZTMP8   zmm26
%define ZTMP9   zmm27
%define ZTMP10  zmm28
%define ZTMP11  zmm29
%define ZTMP12  zmm30
%define ZTMP13  zmm31

struc STACKFRAME
_key_sched:	resq	16*16   ; 16 lanes x 16 qwords; 16 x 128 bytes = 2048
_key_sched2:	resq	16*16   ; 16 lanes x 16 qwords; 16 x 128 bytes = 2048
_key_sched3:	resq	16*16   ; 16 lanes x 16 qwords; 16 x 128 bytes = 2048
_tmp_iv:	resq	16      ; 2 x 64 bytes
_tmp_in:	resq	16      ; 2 x 64 bytes
_tmp_out:	resq	16      ; 2 x 64 bytes
_tmp_mask:	resd	16      ; 1 x 64 bytes
_gpr_save:	resq	4       ; r12 to r15
_rsp_save:	resq	1
_mask_save:	resq	1
_size_save:	resq	1
endstruc

;;; ===========================================================================
;;; ===========================================================================
;;; MACROS
;;; ===========================================================================
;;; ===========================================================================

;;; ===========================================================================
;;; CLEAR TRANSPOSED KEY SCHEDULE (if SAFE_DATA is selected)
;;; ===========================================================================
%macro CLEAR_KEY_SCHEDULE 2
%define %%ALG   %1      ; [in] DES or 3DES
%define %%ZT    %2      ; [clobbered] temporary ZMM register

%ifdef SAFE_DATA
        vpxorq  %%ZT, %%ZT
%assign rep_num (2048 / 64)
%ifidn %%ALG, 3DES
%assign rep_num (rep_num * 3)
%endif

%assign offset 0
%rep rep_num
        vmovdqa64       [rsp + _key_sched + offset], %%ZT
%assign offset (offset + 64)
%endrep

%endif ; SAFE_DATA

%endmacro

;;; ===========================================================================
;;; PERMUTE
;;; ===========================================================================
;;; A [in/out] - zmm register
;;; B [in/out] - zmm register
;;; NSHIFT [in] - constant to shift words by
;;; MASK [in] - zmm or m512 with mask
;;; T0 [clobbered] - temporary zmm register
%macro PERMUTE 5
%define %%A %1
%define %%B %2
%define %%NSHIFT %3
%define %%MASK %4
%define %%T0 %5

        vpsrld  %%T0, %%A, %%NSHIFT
        vpxord  %%T0, %%T0, %%B
        vpandd  %%T0, %%T0, %%MASK
        vpxord  %%B, %%B, %%T0
        vpslld  %%T0, %%T0, %%NSHIFT
        vpxord  %%A, %%A, %%T0
%endmacro

;;; ===========================================================================
;;; INITIAL PERMUTATION
;;; ===========================================================================
;;; L [in/out] - zmm register
;;; R [in/out] - zmm register
;;; T0 [clobbered] - temporary zmm register
%macro IP_Z 3
%define %%L %1
%define %%R %2
%define %%T0 %3
        PERMUTE %%R, %%L, 4,  [rel init_perm_consts + 0*64], %%T0
        PERMUTE %%L, %%R, 16, [rel init_perm_consts + 1*64], %%T0
        PERMUTE %%R, %%L, 2,  [rel init_perm_consts + 2*64], %%T0
        PERMUTE %%L, %%R, 8,  [rel init_perm_consts + 3*64], %%T0
        PERMUTE %%R, %%L, 1,  [rel init_perm_consts + 4*64], %%T0
%endmacro

;;; ===========================================================================
;;; FINAL PERMUTATION
;;; ===========================================================================
;;; L [in/out] - zmm register
;;; R [in/out] - zmm register
;;; T0 [clobbered] - temporary zmm register
%macro FP_Z 3
%define %%L %1
%define %%R %2
%define %%T0 %3
        PERMUTE %%L, %%R, 1,  [rel init_perm_consts + 4*64], %%T0
        PERMUTE %%R, %%L, 8,  [rel init_perm_consts + 3*64], %%T0
        PERMUTE %%L, %%R, 2,  [rel init_perm_consts + 2*64], %%T0
        PERMUTE %%R, %%L, 16, [rel init_perm_consts + 1*64], %%T0
        PERMUTE %%L, %%R, 4,  [rel init_perm_consts + 0*64], %%T0
%endmacro

;;; ===========================================================================
;;; P PHASE
;;; ===========================================================================
;;; W0 [in/out] - zmm register
;;;               in: vector of 16 x 32bits from S phase
;;;               out: permuted in vector
;;; T0-T3 [clobbered] - temporary zmm register
%macro P_PHASE 5
%define %%W0 %1
%define %%T0 %2
%define %%T1 %3
%define %%T2 %4
%define %%T3 %5

        vprord  %%T0, %%W0, 3
	vpandd  %%T0, %%T0, [rel mask_values + 0*64]
        vprord  %%T1, %%W0, 5
	vpandd  %%T1, %%T1, [rel mask_values + 1*64]
	vpord   %%T0, %%T0, %%T1

        vprord  %%T1, %%W0, 24
	vpandd  %%T1, %%T1, [rel mask_values + 2*64]
        vprord  %%T2, %%W0, 26
	vpandd  %%T2, %%T2, [rel mask_values + 3*64]
	vpord   %%T1, %%T1, %%T2
	vpord   %%T0, %%T0, %%T1

        vprord  %%T1, %%W0, 15
	vpandd  %%T1, %%T1, [rel mask_values + 4*64]
        vprord  %%T2, %%W0, 17
	vpandd  %%T2, %%T2, [rel mask_values + 5*64]
	vpord   %%T1, %%T1, %%T2

        vprord  %%T2, %%W0, 6
	vpandd  %%T2, %%T2, [rel mask_values + 6*64]
        vprord  %%T3, %%W0, 21
	vpandd  %%T3, %%T3, [rel mask_values + 7*64]
	vpord   %%T2, %%T2, %%T3
	vpord   %%T1, %%T1, %%T2
	vpord   %%T0, %%T0, %%T1

        vprord  %%T1, %%W0, 12
	vpandd  %%T1, %%T1, [rel mask_values + 8*64]
        vprord  %%T2, %%W0, 14
	vpandd  %%T2, %%T2, [rel mask_values + 9*64]
	vpord   %%T1, %%T1, %%T2

        vprord  %%T2, %%W0, 4
	vpandd  %%T2, %%T2, [rel mask_values + 10*64]
        vprord  %%T3, %%W0, 11
	vpandd  %%T3, %%T3, [rel mask_values + 11*64]
	vpord   %%T2, %%T2, %%T3
	vpord   %%T1, %%T1, %%T2
	vpord   %%T0, %%T0, %%T1

        vprord  %%T1, %%W0, 16
	vpandd  %%T1, %%T1, [rel mask_values + 12*64]
        vprord  %%T2, %%W0, 22
	vpandd  %%T2, %%T2, [rel mask_values + 13*64]
	vpord   %%T1, %%T1, %%T2

        vprord  %%T2, %%W0, 19
	vpandd  %%T2, %%T2, [rel mask_values + 14*64]
        vprord  %%T3, %%W0, 10
	vpandd  %%T3, %%T3, [rel mask_values + 15*64]
	vpord   %%T2, %%T2, %%T3
	vpord   %%T1, %%T1, %%T2
	vpord   %%T0, %%T0, %%T1

        vprord  %%T1, %%W0, 9
	vpandd  %%T1, %%T1, [rel mask_values + 16*64]
        vprord  %%T2, %%W0, 13
	vpandd  %%T2, %%T2, [rel mask_values + 17*64]
	vpord   %%T1, %%T1, %%T2

        vprord  %%T2, %%W0, 25
	vpandd  %%T2, %%T2, [rel mask_values + 18*64]
	vpord   %%T1, %%T1, %%T2
	vpord   %%W0, %%T0, %%T1
%endmacro

;;; ===========================================================================
;;; E PHASE
;;; ===========================================================================
;;;
;;; Expands 16x32-bit words into 16x48-bit words
;;; plus XOR's result with the key schedule.
;;; The output is adjusted to be friendly as S phase input.
;;;
;;; in [in] - zmm register
;;; out0a [out] - zmm register
;;; out0b [out] - zmm register
;;; out1a [out] - zmm register
;;; out1b [out] - zmm register
;;; k0 [in] - key schedule; zmm or m512
;;; k1 [in] - key schedule; zmm or m512
;;; t0-t1 [clobbered] - temporary zmm register
%macro E_PHASE 9
%define %%IN %1
%define %%OUT0A %2
%define %%OUT0B %3
%define %%OUT1A %4
%define %%OUT1B %5
%define %%K0 %6
%define %%K1 %7
%define %%T0 %8
%define %%T1 %9

	vprord          %%T0, %%IN, 31
	vprord          %%T1, %%IN, 3
	vpshufb         %%T0, %%T0, [rel idx_e]
	vpshufb         %%T1, %%T1, [rel idx_e]
	vpunpcklbw      %%OUT0A, %%T0, %%T1
	vpunpckhbw      %%OUT1A, %%T0, %%T1
	vpxord          %%OUT0A, %%OUT0A, %%K0
	vpxord          %%OUT1A, %%OUT1A, %%K1
	vpandd          %%OUT0B, %%OUT0A, [rel and_eu]
	vpsrlw          %%OUT0B, %%OUT0B, 8
	vpandd          %%OUT0A, %%OUT0A, [rel and_ed]
	vpandd          %%OUT1B, %%OUT1A, [rel and_eu]
	vpsrlw          %%OUT1B, %%OUT1B, 8
	vpandd          %%OUT1A, %%OUT1A, [rel and_ed]
%endmacro

;;; ===========================================================================
;;; S-BOX
;;; ===========================================================================
;;;
;;; NOTE: clobbers k1-k6 OpMask registers
;;;
;;; IN0A [in] - zmm register; output from E-phase
;;; IN0B [in] - zmm register; output from E-phase
;;; IN1A [in] - zmm register; output from E-phase
;;; IN1B [in] - zmm register; output from E-phase
;;; OUT [out] - zmm register; output from E-phase
;;; T0-T5 [clobbered] - temporary zmm register
%macro S_PHASE 11
%define %%IN0A %1
%define %%IN0B %2
%define %%IN1A %3
%define %%IN1B %4
%define %%OUT %5
%define %%T0 %6
%define %%T1 %7
%define %%T2 %8
%define %%T3 %9
%define %%T4 %10
%define %%T5 %11

	vmovdqa64       %%T0, [rel reg_values16bit_7]
	vpcmpuw         k3, %%IN0A, %%T0, 2 ; 2 -> LE
	vpcmpuw         k4, %%IN0B, %%T0, 2 ; 2 -> LE
	vpcmpuw         k5, %%IN1A, %%T0, 2 ; 2 -> LE
	vpcmpuw         k6, %%IN1B, %%T0, 2 ; 2 -> LE

	mov             DWORD(IA0), 0x55555555
	kmovd           k1, DWORD(IA0)
	mov             DWORD(IA0), 0xaaaaaaaa
	kmovd           k2, DWORD(IA0)

	vpermw          %%T0{k1}{z}, %%IN0A, [rel S_box_flipped + 0*64]
	vpermw          %%T1{k1}{z}, %%IN0A, [rel S_box_flipped + 1*64]
	vpermw          %%T2{k2}{z}, %%IN0A, [rel S_box_flipped + 4*64]
	vpermw          %%T3{k2}{z}, %%IN0A, [rel S_box_flipped + 5*64]
	vpxord          %%T0, %%T0, %%T2
	vpxord          %%OUT, %%T1, %%T3
        vmovdqu16       %%OUT{k3}, %%T0

	vpermw          %%T0{k1}{z}, %%IN0B, [rel S_box_flipped + 2*64]
	vpermw          %%T1{k1}{z}, %%IN0B, [rel S_box_flipped + 3*64]
	vpermw          %%T2{k2}{z}, %%IN0B, [rel S_box_flipped + 6*64]
	vpermw          %%T3{k2}{z}, %%IN0B, [rel S_box_flipped + 7*64]
	vpxord          %%T0, %%T0, %%T2
	vpxord          %%T3, %%T1, %%T3
        vmovdqu16       %%T3{k4}, %%T0
	vpsllw          %%T3, %%T3, 4
	vpxord          %%OUT, %%OUT, %%T3

	vpermw          %%T0{k1}{z}, %%IN1A, [rel S_box_flipped + 8*64]
	vpermw          %%T1{k1}{z}, %%IN1A, [rel S_box_flipped + 9*64]
	vpermw          %%T2{k2}{z}, %%IN1A, [rel S_box_flipped + 12*64]
	vpermw          %%T3{k2}{z}, %%IN1A, [rel S_box_flipped + 13*64]
	vpxord          %%T0, %%T0, %%T2
	vpxord          %%T4, %%T1, %%T3
        vmovdqu16       %%T4{k5}, %%T0

	vpermw          %%T0{k1}{z}, %%IN1B, [rel S_box_flipped + 10*64]
	vpermw          %%T1{k1}{z}, %%IN1B, [rel S_box_flipped + 11*64]
	vpermw          %%T2{k2}{z}, %%IN1B, [rel S_box_flipped + 14*64]
	vpermw          %%T3{k2}{z}, %%IN1B, [rel S_box_flipped + 15*64]
	vpxord          %%T0, %%T0, %%T2
	vpxord          %%T5, %%T1, %%T3
        vmovdqu16       %%T5{k6}, %%T0
	vpsllw          %%T5, %%T5, 4

	vpxord          %%T4, %%T4, %%T5
	vpsllw          %%T4, %%T4, 8
	vpxord          %%OUT, %%OUT, %%T4
        vpshufb         %%OUT, %%OUT, [rel shuffle_reg]
%endmacro

;;; ===========================================================================
;;; DES encryption/decryption round
;;; ===========================================================================
;;;
;;; Clobbers k1-k6 OpMask registers
;;;
;;; ENC_DEC [in] - ENC for encryption, DEC for decryption
;;; R [in/out] - zmm register; plain text in & cipher text out
;;; L [in/out] - zmm register; plain text in & cipher text out
;;; KS [in] - pointer to the key schedule
;;; T0-T11 [clobbered] - temporary zmm register
%macro DES_ENC_DEC 16
%define %%ENC_DEC %1
%define %%R %2
%define %%L %3
%define %%KS %4
%define %%T0 %5
%define %%T1 %6
%define %%T2 %7
%define %%T3 %8
%define %%T4 %9
%define %%T5 %10
%define %%T6 %11
%define %%T7 %12
%define %%T8 %13
%define %%T9 %14
%define %%T10 %15
%define %%T11 %16

        IP_Z    %%R, %%L, %%T0

%ifidn  %%ENC_DEC, ENC
        ;; ENCRYPTION
        xor     KSOFFSET, KSOFFSET
%%_des_enc_loop:
        E_PHASE %%R, %%T1, %%T2, %%T3, %%T4, [%%KS + KSOFFSET + (0*64)], [%%KS + KSOFFSET + (1*64)], %%T6, %%T7
        S_PHASE %%T1, %%T2, %%T3, %%T4, %%T0, %%T6, %%T7, %%T8, %%T9, %%T10, %%T11
        P_PHASE %%T0, %%T1, %%T2, %%T3, %%T4
        vpxord  %%L, %%L, %%T0

        E_PHASE %%L, %%T1, %%T2, %%T3, %%T4, [%%KS + KSOFFSET + (2*64)], [%%KS + KSOFFSET + (3*64)], %%T6, %%T7
        S_PHASE %%T1, %%T2, %%T3, %%T4, %%T0, %%T6, %%T7, %%T8, %%T9, %%T10, %%T11
        P_PHASE %%T0, %%T1, %%T2, %%T3, %%T4
        vpxord  %%R, %%R, %%T0

        add     KSOFFSET, (4*64)
        cmp     KSOFFSET, (8*(4*64))
        jb      %%_des_enc_loop

%else
        ;; DECRYPTION
        mov     KSOFFSET, (8*(4*64))
%%_des_dec_loop:
        E_PHASE %%R, %%T1, %%T2, %%T3, %%T4, [%%KS + KSOFFSET - (2*64)], [%%KS + KSOFFSET - (1*64)], %%T6, %%T7
        S_PHASE %%T1, %%T2, %%T3, %%T4, %%T0, %%T6, %%T7, %%T8, %%T9, %%T10, %%T11
        P_PHASE %%T0, %%T1, %%T2, %%T3, %%T4
        vpxord  %%L, %%L, %%T0

        E_PHASE %%L, %%T1, %%T2, %%T3, %%T4, [%%KS + KSOFFSET - (4*64)], [%%KS + KSOFFSET - (3*64)], %%T6, %%T7
        S_PHASE %%T1, %%T2, %%T3, %%T4, %%T0, %%T6, %%T7, %%T8, %%T9, %%T10, %%T11
        P_PHASE %%T0, %%T1, %%T2, %%T3, %%T4
        vpxord  %%R, %%R, %%T0
        sub     KSOFFSET, (4*64)
        jnz     %%_des_dec_loop
%endif                          ; DECRYPTION

        FP_Z    %%R, %%L, %%T0
%endmacro

;;; ===========================================================================
;;; DATA TRANSPOSITION AT DATA INPUT
;;; ===========================================================================
;;;
;;; IN00 - IN15 [in/out]:
;;;          in:  IN00 - lane 0 data, IN01 - lane 1 data, ... IN15 - lane 15 data
;;;          out: R0 - 16 x word0, L0 - 16 x word1, ... L7 - 16 x word15
;;; T0-T3 [clobbered] - temporary zmm registers
;;; K0-K5 [clobbered] - temporary zmm registers
;;; H0-H3 [clobbered] - temporary zmm registers
%macro TRANSPOSE_IN 30
%define %%IN00 %1               ; R0
%define %%IN01 %2               ; L0
%define %%IN02 %3               ; R1
%define %%IN03 %4               ; L1
%define %%IN04 %5               ; R2
%define %%IN05 %6               ; L2
%define %%IN06 %7               ; R3
%define %%IN07 %8               ; L3
%define %%IN08 %9               ; R4
%define %%IN09 %10              ; L4
%define %%IN10 %11              ; R5
%define %%IN11 %12              ; L5
%define %%IN12 %13              ; R6
%define %%IN13 %14              ; L6
%define %%IN14 %15              ; R7
%define %%IN15 %16              ; L7
%define %%T0 %17
%define %%T1 %18
%define %%T2 %19
%define %%T3 %20
%define %%K0 %21
%define %%K1 %22
%define %%K2 %23
%define %%K3 %24
%define %%K4 %25
%define %%K5 %26
%define %%H0 %27
%define %%H1 %28
%define %%H2 %29
%define %%H3 %30

	vpunpckldq      %%K0, %%IN00, %%IN01
	vpunpckhdq      %%K1, %%IN00, %%IN01
	vpunpckldq      %%T0, %%IN02, %%IN03
	vpunpckhdq      %%T1, %%IN02, %%IN03

	vpunpckldq      %%IN00, %%IN04, %%IN05
	vpunpckhdq      %%IN01, %%IN04, %%IN05
	vpunpckldq      %%IN02, %%IN06, %%IN07
	vpunpckhdq      %%IN03, %%IN06, %%IN07

        vpunpcklqdq     %%K2, %%K0, %%T0
	vpunpckhqdq     %%T2, %%K0, %%T0
	vpunpcklqdq     %%K3, %%K1, %%T1
	vpunpckhqdq     %%T3, %%K1, %%T1

        vpunpcklqdq     %%K0, %%IN00, %%IN02
	vpunpckhqdq     %%K1, %%IN00, %%IN02
	vpunpcklqdq     %%T0, %%IN01, %%IN03
	vpunpckhqdq     %%T1, %%IN01, %%IN03

	vpunpckldq      %%K4, %%IN08, %%IN09
	vpunpckhdq      %%K5, %%IN08, %%IN09
	vpunpckldq      %%IN04, %%IN10, %%IN11
	vpunpckhdq      %%IN05, %%IN10, %%IN11
	vpunpckldq      %%IN06, %%IN12, %%IN13
	vpunpckhdq      %%IN07, %%IN12, %%IN13
	vpunpckldq      %%IN10, %%IN14, %%IN15
	vpunpckhdq      %%IN11, %%IN14, %%IN15

        vpunpcklqdq     %%IN12, %%K4, %%IN04
	vpunpckhqdq     %%IN13, %%K4, %%IN04
        vpunpcklqdq     %%IN14, %%K5, %%IN05
	vpunpckhqdq     %%IN15, %%K5, %%IN05
        vpunpcklqdq     %%IN00, %%IN06, %%IN10
	vpunpckhqdq     %%IN01, %%IN06, %%IN10
        vpunpcklqdq     %%IN02, %%IN07, %%IN11
	vpunpckhqdq     %%IN03, %%IN07, %%IN11

        vshufi64x2      %%H0, %%K2, %%K0, 0x44
        vshufi64x2      %%H1, %%K2, %%K0, 0xee
        vshufi64x2      %%H2, %%IN12, %%IN00, 0x44
        vshufi64x2      %%H3, %%IN12, %%IN00, 0xee
        vshufi64x2      %%IN00, %%H0, %%H2, 0x88    ; R0
        vshufi64x2      %%IN04, %%H0, %%H2, 0xdd    ; R2
        vshufi64x2      %%IN08, %%H1, %%H3, 0x88    ; R4
        vshufi64x2      %%IN12, %%H1, %%H3, 0xdd    ; R6

        vshufi64x2      %%H0, %%T2, %%K1, 0x44
        vshufi64x2      %%H1, %%T2, %%K1, 0xee
        vshufi64x2      %%H2, %%IN13, %%IN01, 0x44
        vshufi64x2      %%H3, %%IN13, %%IN01, 0xee
        vshufi64x2      %%IN01, %%H0, %%H2, 0x88    ; L0
        vshufi64x2      %%IN05, %%H0, %%H2, 0xdd    ; L2
        vshufi64x2      %%IN09, %%H1, %%H3, 0x88    ; L4
        vshufi64x2      %%IN13, %%H1, %%H3, 0xdd    ; L6

        vshufi64x2      %%H0, %%K3, %%T0, 0x44
        vshufi64x2      %%H1, %%K3, %%T0, 0xee
        vshufi64x2      %%H2, %%IN14, %%IN02, 0x44
        vshufi64x2      %%H3, %%IN14, %%IN02, 0xee
        vshufi64x2      %%IN02, %%H0, %%H2, 0x88    ; R1
        vshufi64x2      %%IN06, %%H0, %%H2, 0xdd    ; R3
        vshufi64x2      %%IN10, %%H1, %%H3, 0x88    ; R5
        vshufi64x2      %%IN14, %%H1, %%H3, 0xdd    ; R7

        vshufi64x2      %%H0, %%T3, %%T1, 0x44
        vshufi64x2      %%H1, %%T3, %%T1, 0xee
        vshufi64x2      %%H2, %%IN15, %%IN03, 0x44
        vshufi64x2      %%H3, %%IN15, %%IN03, 0xee
        vshufi64x2      %%IN03, %%H0, %%H2, 0x88    ; L1
        vshufi64x2      %%IN07, %%H0, %%H2, 0xdd    ; L3
        vshufi64x2      %%IN11, %%H1, %%H3, 0x88    ; L5
        vshufi64x2      %%IN15, %%H1, %%H3, 0xdd    ; L7
%endmacro

;;; ===========================================================================
;;; DATA TRANSPOSITION AT DATA OUTPUT
;;; ===========================================================================
;;;
;;; IN00-IN15 aka R0/L0 - R7/L7 [in/out]:
;;;          in:  R0 - 16 x word0, L0 - 16 x word1, ... L7 - 16 x word15
;;;          out: R0 - lane 0 data, L0 - lane 1 data, ... L7 - lane 15 data
;;; T0-T3 [clobbered] - temporary zmm registers
;;; K0-K5 [clobbered] - temporary zmm registers
;;; H0-H3 [clobbered] - temporary zmm registers
%macro TRANSPOSE_OUT 30
%define %%IN00 %1               ; R0
%define %%IN01 %2               ; L0
%define %%IN02 %3               ; R1
%define %%IN03 %4               ; L1
%define %%IN04 %5               ; R2
%define %%IN05 %6               ; L2
%define %%IN06 %7               ; R3
%define %%IN07 %8               ; L3
%define %%IN08 %9               ; R4
%define %%IN09 %10              ; L4
%define %%IN10 %11              ; R5
%define %%IN11 %12              ; L5
%define %%IN12 %13              ; R6
%define %%IN13 %14              ; L6
%define %%IN14 %15              ; R7
%define %%IN15 %16              ; L7
%define %%T0 %17
%define %%T1 %18
%define %%T2 %19
%define %%T3 %20
%define %%K0 %21
%define %%K1 %22
%define %%K2 %23
%define %%K3 %24
%define %%K4 %25
%define %%K5 %26
%define %%H0 %27
%define %%H1 %28
%define %%H2 %29
%define %%H3 %30

	vpunpckldq      %%K0, %%IN01, %%IN00
	vpunpckhdq      %%K1, %%IN01, %%IN00
	vpunpckldq      %%T0, %%IN03, %%IN02
	vpunpckhdq      %%T1, %%IN03, %%IN02

	vpunpckldq      %%IN00, %%IN05, %%IN04
	vpunpckhdq      %%IN01, %%IN05, %%IN04
	vpunpckldq      %%IN02, %%IN07, %%IN06
	vpunpckhdq      %%IN03, %%IN07, %%IN06

        vpunpcklqdq     %%K2, %%K0, %%T0
	vpunpckhqdq     %%T2, %%K0, %%T0
	vpunpcklqdq     %%K3, %%K1, %%T1
	vpunpckhqdq     %%T3, %%K1, %%T1

        vpunpcklqdq     %%K0, %%IN00, %%IN02
	vpunpckhqdq     %%K1, %%IN00, %%IN02
	vpunpcklqdq     %%T0, %%IN01, %%IN03
	vpunpckhqdq     %%T1, %%IN01, %%IN03

	vpunpckldq      %%K4, %%IN09, %%IN08
	vpunpckhdq      %%K5, %%IN09, %%IN08
	vpunpckldq      %%IN04, %%IN11, %%IN10
	vpunpckhdq      %%IN05, %%IN11, %%IN10
	vpunpckldq      %%IN06, %%IN13, %%IN12
	vpunpckhdq      %%IN07, %%IN13, %%IN12
	vpunpckldq      %%IN10, %%IN15, %%IN14
	vpunpckhdq      %%IN11, %%IN15, %%IN14

        vpunpcklqdq     %%IN12, %%K4, %%IN04
	vpunpckhqdq     %%IN13, %%K4, %%IN04
        vpunpcklqdq     %%IN14, %%K5, %%IN05
	vpunpckhqdq     %%IN15, %%K5, %%IN05
        vpunpcklqdq     %%IN00, %%IN06, %%IN10
	vpunpckhqdq     %%IN01, %%IN06, %%IN10
        vpunpcklqdq     %%IN02, %%IN07, %%IN11
	vpunpckhqdq     %%IN03, %%IN07, %%IN11

        vshufi64x2      %%H0, %%K2, %%K0, 0x44
        vshufi64x2      %%H1, %%K2, %%K0, 0xee
        vshufi64x2      %%H2, %%IN12, %%IN00, 0x44
        vshufi64x2      %%H3, %%IN12, %%IN00, 0xee
        vshufi64x2      %%IN00, %%H0, %%H2, 0x88    ; R0
        vshufi64x2      %%IN04, %%H0, %%H2, 0xdd    ; R2
        vshufi64x2      %%IN08, %%H1, %%H3, 0x88    ; R4
        vshufi64x2      %%IN12, %%H1, %%H3, 0xdd    ; R6

        vshufi64x2      %%H0, %%T2, %%K1, 0x44
        vshufi64x2      %%H1, %%T2, %%K1, 0xee
        vshufi64x2      %%H2, %%IN13, %%IN01, 0x44
        vshufi64x2      %%H3, %%IN13, %%IN01, 0xee
        vshufi64x2      %%IN01, %%H0, %%H2, 0x88    ; L0
        vshufi64x2      %%IN05, %%H0, %%H2, 0xdd    ; L2
        vshufi64x2      %%IN09, %%H1, %%H3, 0x88    ; L4
        vshufi64x2      %%IN13, %%H1, %%H3, 0xdd    ; L6

        vshufi64x2      %%H0, %%K3, %%T0, 0x44
        vshufi64x2      %%H1, %%K3, %%T0, 0xee
        vshufi64x2      %%H2, %%IN14, %%IN02, 0x44
        vshufi64x2      %%H3, %%IN14, %%IN02, 0xee
        vshufi64x2      %%IN02, %%H0, %%H2, 0x88    ; R1
        vshufi64x2      %%IN06, %%H0, %%H2, 0xdd    ; R3
        vshufi64x2      %%IN10, %%H1, %%H3, 0x88    ; R5
        vshufi64x2      %%IN14, %%H1, %%H3, 0xdd    ; R7

        vshufi64x2      %%H0, %%T3, %%T1, 0x44
        vshufi64x2      %%H1, %%T3, %%T1, 0xee
        vshufi64x2      %%H2, %%IN15, %%IN03, 0x44
        vshufi64x2      %%H3, %%IN15, %%IN03, 0xee
        vshufi64x2      %%IN03, %%H0, %%H2, 0x88    ; L1
        vshufi64x2      %%IN07, %%H0, %%H2, 0xdd    ; L3
        vshufi64x2      %%IN11, %%H1, %%H3, 0x88    ; L5
        vshufi64x2      %%IN15, %%H1, %%H3, 0xdd    ; L7
%endmacro

;;; ===========================================================================
;;; DATA TRANSPOSITION OF ONE DES BLOCK AT DATA INPUT
;;; ===========================================================================
;;;
;;; IN00-IN15 / R0/L0-R7/L7 [in/out]:
;;;          in:  IN00 - lane 0 data, IN01 - lane 1 data, ... IN15 - lane 15 data
;;;          out: R0 - 16 x word0, L0 - 16 x word1
;;; T0,T2 [clobbered] - temporary zmm registers
;;; K0-K4 [clobbered] - temporary zmm registers
;;; H0,H2 [clobbered] - temporary zmm registers
%macro TRANSPOSE_IN_ONE 24
%define %%IN00 %1               ; R0
%define %%IN01 %2               ; L0
%define %%IN02 %3               ; R1
%define %%IN03 %4               ; L1
%define %%IN04 %5               ; R2
%define %%IN05 %6               ; L2
%define %%IN06 %7               ; R3
%define %%IN07 %8               ; L3
%define %%IN08 %9               ; R4
%define %%IN09 %10              ; L4
%define %%IN10 %11              ; R5
%define %%IN11 %12              ; L5
%define %%IN12 %13              ; R6
%define %%IN13 %14              ; L6
%define %%IN14 %15              ; R7
%define %%IN15 %16              ; L7
%define %%T0 %17
%define %%T2 %18
%define %%K0 %19
%define %%K1 %20
%define %%K2 %21
%define %%K4 %22
%define %%H0 %23
%define %%H2 %24

	vpunpckldq      %%K0, %%IN00, %%IN01
	vpunpckhdq      %%K1, %%IN00, %%IN01
	vpunpckldq      %%T0, %%IN02, %%IN03

	vpunpckldq      %%IN00, %%IN04, %%IN05
	vpunpckhdq      %%IN01, %%IN04, %%IN05
	vpunpckldq      %%IN02, %%IN06, %%IN07

        vpunpcklqdq     %%K2, %%K0, %%T0
	vpunpckhqdq     %%T2, %%K0, %%T0

        vpunpcklqdq     %%K0, %%IN00, %%IN02
	vpunpckhqdq     %%K1, %%IN00, %%IN02

	vpunpckldq      %%K4, %%IN08, %%IN09
	vpunpckldq      %%IN04, %%IN10, %%IN11
	vpunpckldq      %%IN06, %%IN12, %%IN13
	vpunpckldq      %%IN10, %%IN14, %%IN15

        vpunpcklqdq     %%IN12, %%K4, %%IN04
	vpunpckhqdq     %%IN13, %%K4, %%IN04
        vpunpcklqdq     %%IN00, %%IN06, %%IN10
	vpunpckhqdq     %%IN01, %%IN06, %%IN10

        vshufi64x2      %%H0, %%K2, %%K0, 0x44
        vshufi64x2      %%H2, %%IN12, %%IN00, 0x44
        vshufi64x2      %%IN00, %%H0, %%H2, 0x88    ; R0

        vshufi64x2      %%H0, %%T2, %%K1, 0x44
        vshufi64x2      %%H2, %%IN13, %%IN01, 0x44
        vshufi64x2      %%IN01, %%H0, %%H2, 0x88    ; L0
%endmacro

;;; ===========================================================================
;;; DATA TRANSPOSITION OF ONE DES BLOCK AT DATA OUTPUT
;;; ===========================================================================
;;;
;;; IN00-IN15 aka R0/L0 - R7/L7 [in/out]:
;;;          in:  R0 - 16 x word0, L0 - 16 x word1
;;;          out: R0 - lane 0 data, L0 - lane 1 data, ... L7 - lane 15 data
;;; T0-T3 [clobbered] - temporary zmm registers
;;; K0-K3 [clobbered] - temporary zmm registers
;;; H0,H1 [clobbered] - temporary zmm registers
%macro TRANSPOSE_OUT_ONE 25
%define %%IN00 %1               ; R0
%define %%IN01 %2               ; L0
%define %%IN02 %3               ; R1
%define %%IN03 %4               ; L1
%define %%IN04 %5               ; R2
%define %%IN05 %6               ; L2
%define %%IN06 %7               ; R3
%define %%IN07 %8               ; L3
%define %%IN08 %9               ; R4
%define %%IN09 %10              ; L4
%define %%IN10 %11              ; R5
%define %%IN11 %12              ; L5
%define %%IN12 %13              ; R6
%define %%IN13 %14              ; L6
%define %%IN14 %15              ; R7
%define %%IN15 %16              ; L7
%define %%T0 %17
%define %%T2 %18
%define %%T3 %19
%define %%K0 %20
%define %%K1 %21
%define %%K2 %22
%define %%K3 %23
%define %%H0 %24
%define %%H1 %25

        vpxord          %%T0, %%T0, %%T0

	vpunpckldq      %%K0, %%IN01, %%IN00
	vpunpckhdq      %%K1, %%IN01, %%IN00

        vpunpcklqdq     %%K2, %%K0, %%T0
	vpunpckhqdq     %%T2, %%K0, %%T0
	vpunpcklqdq     %%K3, %%K1, %%T0
	vpunpckhqdq     %%T3, %%K1, %%T0

        vshufi64x2      %%H0, %%K2, %%T0, 0x44
        vshufi64x2      %%H1, %%K2, %%T0, 0xee
        vshufi64x2      %%IN00, %%H0, %%T0, 0x88    ; R0
        vshufi64x2      %%IN04, %%H0, %%T0, 0xdd    ; R2
        vshufi64x2      %%IN08, %%H1, %%T0, 0x88    ; R4
        vshufi64x2      %%IN12, %%H1, %%T0, 0xdd    ; R6

        vshufi64x2      %%H0, %%T2, %%T0, 0x44
        vshufi64x2      %%H1, %%T2, %%T0, 0xee
        vshufi64x2      %%IN01, %%H0, %%T0, 0x88    ; L0
        vshufi64x2      %%IN05, %%H0, %%T0, 0xdd    ; L2
        vshufi64x2      %%IN09, %%H1, %%T0, 0x88    ; L4
        vshufi64x2      %%IN13, %%H1, %%T0, 0xdd    ; L6

        vshufi64x2      %%H0, %%K3, %%T0, 0x44
        vshufi64x2      %%H1, %%K3, %%T0, 0xee
        vshufi64x2      %%IN02, %%H0, %%T0, 0x88    ; R1
        vshufi64x2      %%IN06, %%H0, %%T0, 0xdd    ; R3
        vshufi64x2      %%IN10, %%H1, %%T0, 0x88    ; R5
        vshufi64x2      %%IN14, %%H1, %%T0, 0xdd    ; R7

        vshufi64x2      %%H0, %%T3, %%T0, 0x44
        vshufi64x2      %%H1, %%T3, %%T0, 0xee
        vshufi64x2      %%IN03, %%H0, %%T0, 0x88    ; L1
        vshufi64x2      %%IN07, %%H0, %%T0, 0xdd    ; L3
        vshufi64x2      %%IN11, %%H1, %%T0, 0x88    ; L5
        vshufi64x2      %%IN15, %%H1, %%T0, 0xdd    ; L7
%endmacro

;;; ===========================================================================
;;; DES INITIALIZATION
;;; key schedule transposition and IV set up
;;; ===========================================================================
;;;
;;; STATE_KEYS [in]  - KEYS in DES OOO STATE
;;; STATE_IV [ in]  - IV in DES OOO STATE
;;; KS    [out] - place to store transposed key schedule or NULL
;;; IV0   [out] - r512; initialization vector
;;; IV1   [out] - r512; initialization vector
;;; T0-T27 [clobbered] - temporary r512
%macro DES_INIT 33
%define %%STATE_KEYS %1
%define %%STATE_IV   %2
%define %%KS   %3
%define %%IV0  %4
%define %%IV1  %5
%define %%T0   %6
%define %%T1   %7
%define %%T2   %8
%define %%T3   %9
%define %%T4   %10
%define %%T5   %11
%define %%T6   %12
%define %%T7   %13
%define %%T8   %14
%define %%T9   %15
%define %%T10  %16
%define %%T11  %17
%define %%T12  %18
%define %%T13  %19
%define %%T14  %20
%define %%T15  %21
%define %%T16  %22
%define %%T17  %23
%define %%T18  %24
%define %%T19  %25
%define %%T20  %26
%define %%T21  %27
%define %%T22  %28
%define %%T23  %29
%define %%T24  %30
%define %%T25  %31
%define %%T26  %32
%define %%T27  %33

        ;; set up the key schedule
        ;; - load first half of the keys & transpose
        ;; - transpose and store
        ;; note: we can use IV registers as temprary ones here
%assign IDX 0
%rep 16
        mov             IA0, [%%STATE_KEYS + (IDX*PTR_SZ)]
        vmovdqu64       %%T %+ IDX, [IA0]
%assign IDX (IDX + 1)
%endrep
        TRANSPOSE_IN %%T0, %%T1, %%T2, %%T3, %%T4, %%T5, %%T6, %%T7, %%T8, %%T9, %%T10, %%T11, %%T12, %%T13, %%T14, %%T15, %%T16, %%T17, %%T18, %%T19, %%T20, %%T21, %%T22, %%T23, %%T24, %%T25, %%T26, %%T27, %%IV0, %%IV1
%assign IDX 0
%rep 16
        vmovdqu64       [%%KS + (IDX * 64)], %%T %+ IDX
%assign IDX (IDX + 1)
%endrep
        ;; - load second half of the keys & transpose
        ;; - transpose and store
        ;; note: we can use IV registers as temprary ones here
%assign IDX 0
%rep 16
        mov             IA0, [%%STATE_KEYS + (IDX*PTR_SZ)]
        vmovdqu64       %%T %+ IDX, [IA0 + 64]
%assign IDX (IDX + 1)
%endrep
        TRANSPOSE_IN %%T0, %%T1, %%T2, %%T3, %%T4, %%T5, %%T6, %%T7, %%T8, %%T9, %%T10, %%T11, %%T12, %%T13, %%T14, %%T15, %%T16, %%T17, %%T18, %%T19, %%T20, %%T21, %%T22, %%T23, %%T24, %%T25, %%T26, %%T27, %%IV0, %%IV1
%assign IDX 0
%rep 16
        vmovdqu64       [%%KS + (16 * 64) + (IDX * 64)], %%T %+ IDX
%assign IDX (IDX + 1)
%endrep

        ;; set up IV
        ;; - they are already kept transposed so this is enough to load them
        vmovdqu64       %%IV0, [%%STATE_IV + (0 * 64)]
        vmovdqu64       %%IV1, [%%STATE_IV + (1 * 64)]
%endmacro

;;; ===========================================================================
;;; 3DES INITIALIZATION
;;; key schedule transposition and IV set up
;;; ===========================================================================
;;;
;;; STATE_KEYS [in]  - KEYS in 3DES OOO STATE
;;; STATE_IV [ in]  - IV in 3DES OOO STATE
;;; KS1   [out] - place to store transposed key schedule or NULL
;;; KS2   [out] - place to store transposed key schedule or NULL
;;; KS3   [out] - place to store transposed key schedule or NULL
;;; IV0   [out] - r512; initialization vector
;;; IV1   [out] - r512; initialization vector
;;; T0-T27 [clobbered] - temporary r512
;;; DIR   [in] - ENC/DEC (keys arranged in different order for enc/dec)
%macro DES3_INIT 36
%define %%STATE_KEYS %1
%define %%STATE_IV   %2
%define %%KS1  %3
%define %%KS2  %4
%define %%KS3  %5
%define %%IV0  %6
%define %%IV1  %7
%define %%T0   %8
%define %%T1   %9
%define %%T2   %10
%define %%T3   %11
%define %%T4   %12
%define %%T5   %13
%define %%T6   %14
%define %%T7   %15
%define %%T8   %16
%define %%T9   %17
%define %%T10  %18
%define %%T11  %19
%define %%T12  %20
%define %%T13  %21
%define %%T14  %22
%define %%T15  %23
%define %%T16  %24
%define %%T17  %25
%define %%T18  %26
%define %%T19  %27
%define %%T20  %28
%define %%T21  %29
%define %%T22  %30
%define %%T23  %31
%define %%T24  %32
%define %%T25  %33
%define %%T26  %34
%define %%T27  %35
%define %%DIR  %36

%ifidn %%DIR, ENC
%assign KEY_IDX 0
%else
%assign KEY_IDX 2
%endif
%assign KS_IDX  1

%rep 3
        ;; set up the key schedule
        ;; - load first half of the keys & transpose
        ;; - transpose and store
        ;; note: we can use IV registers as temprary ones here

%assign IDX 0
%rep 16
        mov             IA0, [%%STATE_KEYS + (IDX*PTR_SZ)]
        mov             IA0, [IA0 + (KEY_IDX * PTR_SZ)]
        vmovdqu64       %%T %+ IDX, [IA0]
%assign IDX (IDX + 1)
%endrep
        TRANSPOSE_IN %%T0, %%T1, %%T2, %%T3, %%T4, %%T5, %%T6, %%T7, %%T8, %%T9, %%T10, %%T11, %%T12, %%T13, %%T14, %%T15, %%T16, %%T17, %%T18, %%T19, %%T20, %%T21, %%T22, %%T23, %%T24, %%T25, %%T26, %%T27, %%IV0, %%IV1
%assign IDX 0
%rep 16
        vmovdqu64       [%%KS %+ KS_IDX + (IDX * 64)], %%T %+ IDX
%assign IDX (IDX + 1)
%endrep
        ;; - load second half of the keys & transpose
        ;; - transpose and store
        ;; note: we can use IV registers as temprary ones here
%assign IDX 0
%rep 16
        mov             IA0, [%%STATE_KEYS + (IDX*PTR_SZ)]
        mov             IA0, [IA0 + (KEY_IDX * PTR_SZ)]
        vmovdqu64       %%T %+ IDX, [IA0 + 64]
%assign IDX (IDX + 1)
%endrep
        TRANSPOSE_IN %%T0, %%T1, %%T2, %%T3, %%T4, %%T5, %%T6, %%T7, %%T8, %%T9, %%T10, %%T11, %%T12, %%T13, %%T14, %%T15, %%T16, %%T17, %%T18, %%T19, %%T20, %%T21, %%T22, %%T23, %%T24, %%T25, %%T26, %%T27, %%IV0, %%IV1
%assign IDX 0
%rep 16
        vmovdqu64       [%%KS %+ KS_IDX + (16 * 64) + (IDX * 64)], %%T %+ IDX
%assign IDX (IDX + 1)
%endrep

%ifidn %%DIR, ENC
%assign KEY_IDX (KEY_IDX + 1)
%else
%assign KEY_IDX (KEY_IDX - 1)
%endif
%assign KS_IDX (KS_IDX + 1)
%endrep                         ; KEY_IDX / KS_IDX

        ;; set up IV
        ;; - they are already kept transposed so this is enough to load them
        vmovdqu64       %%IV0, [%%STATE_IV + (0 * 64)]
        vmovdqu64       %%IV1, [%%STATE_IV + (1 * 64)]

%endmacro

;;; ===========================================================================
;;; DES FINISH
;;; Update in/out pointers and store IV
;;; ===========================================================================
;;;
;;; Needs: STATE & SIZE
;;; IV0   [in] - r512; initialization vector
;;; IV1   [in] - r512; initialization vector
;;; T0-T4 [clobbered] - temporary r512 registers
%macro DES_FINISH 7
%define %%IV0   %1
%define %%IV1   %2
%define %%T0    %3
%define %%T1    %4
%define %%T2    %5
%define %%T3    %6
%define %%T4    %7

        vpbroadcastq    %%T4, SIZE
        vmovdqu64       %%T0, [STATE + _des_args_in + (0 * PTR_SZ)]
        vmovdqu64       %%T1, [STATE + _des_args_in + (8 * PTR_SZ)]
        vmovdqu64       %%T2, [STATE + _des_args_out + (0 * PTR_SZ)]
        vmovdqu64       %%T3, [STATE + _des_args_out + (8 * PTR_SZ)]
        vpaddq          %%T0, %%T0, %%T4
        vpaddq          %%T1, %%T1, %%T4
        vpaddq          %%T2, %%T2, %%T4
        vpaddq          %%T3, %%T3, %%T4
        vmovdqu64       [STATE + _des_args_in + (0 * PTR_SZ)], %%T0
        vmovdqu64       [STATE + _des_args_in + (8 * PTR_SZ)], %%T1
        vmovdqu64       [STATE + _des_args_out + (0 * PTR_SZ)], %%T2
        vmovdqu64       [STATE + _des_args_out + (8 * PTR_SZ)], %%T3

        vmovdqu64       [STATE + _des_args_IV + (0 * 64)], %%IV0
        vmovdqu64       [STATE + _des_args_IV + (1 * 64)], %%IV1
%endmacro

;;; ===========================================================================
;;; DES CFB ENCRYPT/DECRYPT - ONE BLOCK ONLY
;;; ===========================================================================
;;;
;;; Needs: STATE, IA0-IA2
;;; ENC_DEC [in] - encyrpt (ENC) or decrypt (DEC) selection
;;; KS      [in] - key schedule
;;; T0-T24  [clobbered] - temporary r512
;;; T_IN    [in] - 16 * 8 byte storage
;;; T_OUT   [in] - 16 * 8 byte storage
;;; T_MASK  [in] - 16 * 4 byte storage
;;; T_IV    [in] - 16 * 8 byte storage
;;;
;;; NOTE: clobbers OpMask registers
%macro DES_CFB_ONE 31
%define %%ENC_DEC %1
%define %%KS      %2
%define %%T0      %3
%define %%T1      %4
%define %%T2      %5
%define %%T3      %6
%define %%T4      %7
%define %%T5      %8
%define %%T6      %9
%define %%T7      %10
%define %%T8      %11
%define %%T9      %12
%define %%T10     %13
%define %%T11     %14
%define %%T12     %15
%define %%T13     %16
%define %%T14     %17
%define %%T15     %18
%define %%T16     %19
%define %%T17     %20
%define %%T18     %21
%define %%T19     %22
%define %%T20     %23
%define %%T21     %24
%define %%T22     %25
%define %%T23     %26
%define %%T24     %27
%define %%T_IN    %28
%define %%T_OUT   %29
%define %%T_IV    %30
%define %%T_MASK  %31

        ;; - find mask for non-zero partial lengths
        vpxord          %%T10, %%T10, %%T10
        vmovdqu64       %%T0, [STATE + _des_args_PLen]
        vpcmpd          k3, %%T0, %%T10, 4 ; NEQ
        kmovw           DWORD(IA0), k3
        movzx           DWORD(IA0), WORD(IA0)
        or              DWORD(IA0), DWORD(IA0)
        jz              %%_des_cfb_one_end ; no non-zero partial lengths

%ifidn %%ENC_DEC, ENC
        ;; For encyrption case we need to make sure that
        ;; all full blocks are complete before proceeding
        ;; with CFB partial block.
        ;; To do that current out position is compared against
        ;; calculated last full block position.
        vmovdqu64       %%T1, [STATE + _des_args_out + (0*8)]
        vmovdqu64       %%T2, [STATE + _des_args_LOut + (0*8)]
        vmovdqu64       %%T3, [STATE + _des_args_out + (8*8)]
        vmovdqu64       %%T4, [STATE + _des_args_LOut + (8*8)]
        vpcmpq          k4, %%T1, %%T2, 0 ; EQ
        vpcmpq          k5, %%T3, %%T4, 0 ; EQ
        kmovw           DWORD(IA1), k4
        movzx           DWORD(IA1), BYTE(IA1)
        kmovw           DWORD(IA2), k5
        movzx           DWORD(IA2), BYTE(IA2)
        shl             DWORD(IA2), 8
        or              DWORD(IA2), DWORD(IA1)
        and             DWORD(IA0), DWORD(IA2)
        jz              %%_des_cfb_one_end ; no non-zero lengths left
        kmovw           k3, DWORD(IA0)
%endif
        ;; Calculate ((1 << partial_bytes) - 1)
        ;; in order to get the mask for loads and stores
        ;; k3 & IA0 - hold valid mask
        vmovdqa64       %%T1, [rel vec_ones_32b]
        vpsllvd         %%T2{k3}{z}, %%T1, %%T0
        vpsubd          %%T2{k3}{z}, %%T2, %%T1
        vmovdqu64       [%%T_MASK], %%T2

        ;; clear selected partial lens not to do them twice
        vmovdqu32       [STATE + _des_args_PLen]{k3}, %%T10

        ;; copy IV, in and out pointers
        vmovdqu64       %%T1, [STATE + _des_args_in + (0*PTR_SZ)]
        vmovdqu64       %%T2, [STATE + _des_args_in + (8*PTR_SZ)]
        vmovdqu64       %%T3, [STATE + _des_args_out + (0*PTR_SZ)]
        vmovdqu64       %%T4, [STATE + _des_args_out + (8*PTR_SZ)]
        vmovdqu64       %%T5, [STATE + _des_args_IV + (0*64)]
        vmovdqu64       %%T6, [STATE + _des_args_IV + (1*64)]
        vmovdqu64       [%%T_IN + (0*PTR_SZ)], %%T1
        vmovdqu64       [%%T_IN + (8*PTR_SZ)], %%T2
        vmovdqu64       [%%T_OUT + (0*PTR_SZ)], %%T3
        vmovdqu64       [%%T_OUT + (8*PTR_SZ)], %%T4
        vmovdqu64       [%%T_IV + (0*64)], %%T5
        vmovdqu64       [%%T_IV + (1*64)], %%T6

        ;; calculate last block case mask
        ;; - first block case requires no modifications to in/out/IV
        vmovdqu64       %%T1, [STATE + _des_args_BLen]
        vpcmpd          k2, %%T1, %%T10, 4 ; NEQ
        kmovw           DWORD(IA1), k2
        and             DWORD(IA1), DWORD(IA0)
        jz              %%_des_cfb_one_no_last_blocks

        ;; set up IV, in and out for the last block case
        ;; - Last block needs in and out to be set differently (decryption only)
        ;; - IA1 holds the last block mask
%ifidn %%ENC_DEC, DEC
        mov             DWORD(IA0), DWORD(IA1)
        mov             DWORD(IA2), DWORD(IA1)
        shr             DWORD(IA1), 8
        and             DWORD(IA2), 0xff
        kmovw           k4, DWORD(IA2)
        kmovw           k5, DWORD(IA1)
        vmovdqu64       %%T1, [STATE + _des_args_LOut + (0*PTR_SZ)]
        vmovdqu64       %%T2, [STATE + _des_args_LOut + (8*PTR_SZ)]
        vmovdqu64       %%T3, [STATE + _des_args_LIn + (0*PTR_SZ)]
        vmovdqu64       %%T4, [STATE + _des_args_LIn + (8*PTR_SZ)]
        vmovdqu64       [%%T_OUT + (0*PTR_SZ)]{k4}, %%T1
        vmovdqu64       [%%T_OUT + (8*PTR_SZ)]{k5}, %%T2
        vmovdqu64       [%%T_IN + (0*PTR_SZ)]{k4}, %%T3
        vmovdqu64       [%%T_IN + (8*PTR_SZ)]{k5}, %%T4
%endif                          ; decryption
        ;; - IV has to be set differently for CFB as well
        ;; - IA0 holds the last block mask
%assign IDX 0
%rep 16
        test            DWORD(IA0), (1 << IDX)
        jz              %%_des_cfb_one_copy_iv_next %+ IDX
%ifidn %%ENC_DEC, ENC
        mov             IA2, [STATE + _des_args_LOut + (IDX*PTR_SZ)]
%else
        mov             IA2, [STATE + _des_args_LIn + (IDX*PTR_SZ)]
%endif
        mov             IA2, [IA2 - 8]
        mov             [%%T_IV + (0*4) + (IDX*4)], DWORD(IA2)
        shr             IA2, 32
        mov             [%%T_IV + (16*4) + (IDX*4)], DWORD(IA2)
%%_des_cfb_one_copy_iv_next %+ IDX:
%assign IDX (IDX + 1)
%endrep

%%_des_cfb_one_no_last_blocks:
        ;; Uffff ... finally let's do some DES CFB
        ;; - let's use T_IN, T_OUT, T_IV and T_MASK

        ;; - load data with the corresponding masks & transpose
        ;; - T0 to T15 will hold the data
        xor             IA0, IA0
%assign IDX 0
%assign K_IDX 1
%rep 16
        mov             IA1, [%%T_IN + (IDX*PTR_SZ)]
        mov             DWORD(IA0), [%%T_MASK + (IDX*4)]
        kmovq           k %+ K_IDX, IA0
        vmovdqu8        %%T %+ IDX{k %+ K_IDX}{z}, [IA1]
%assign IDX (IDX + 1)
%assign K_IDX (K_IDX + 1)
%if K_IDX > 7
%assign K_IDX 1                 ; iterate through K1 to K7
%endif
%endrep
        ;; - transpose the data in T0 to T15, T16 to T23 are clobbered
        TRANSPOSE_IN_ONE %%T0, %%T1, %%T2, %%T3, %%T4, %%T5, %%T6, %%T7, %%T8, %%T9, %%T10, %%T11, %%T12, %%T13, %%T14, %%T15, %%T16, %%T17, %%T18, %%T19, %%T20, %%T21, %%T22, %%T23

        ;; - set up IV and %%T16 & %%T17 used as IV0 and IV1
        vmovdqu64       %%T16, [%%T_IV + (0 * 64)] ;IV0
        vmovdqu64       %%T17, [%%T_IV + (1 * 64)] ;IV1
        ;; DES encrypt
        ;; - R0 - %%T0
        ;; - L0 - %%T1
        DES_ENC_DEC     ENC, %%T16, %%T17, %%KS, %%T2, %%T3, %%T4, %%T5, %%T6, %%T7, %%T8, %%T9, %%T10, %%T11, %%T12, %%T13
        ;; CFB style xor with R0/L0 with IV
        ;; - IV0 - %%T16
        ;; - IV1 - %%T17
        vpxord          %%T2, %%T17, %%T0 ; R0 ^ IV1
        vpxord          %%T0, %%T16, %%T1 ; L0 ^ IV0
        vmovdqa64       %%T1, %%T2
        ;; - new R0 = L0 ^ IV0 (%%T0)
        ;; - new L0 = R0 ^ IV1 (%%T1)

        ;; Transpose the data out
        ;; - %%T2 to %%T24 clobbered
        TRANSPOSE_OUT_ONE %%T0, %%T1, %%T2, %%T3, %%T4, %%T5, %%T6, %%T7, %%T8, %%T9, %%T10, %%T11, %%T12, %%T13, %%T14, %%T15, %%T16, %%T17, %%T18, %%T19, %%T20, %%T21, %%T22, %%T23, %%T24

        ;; Store the transposed data
        ;; - T0 to T15 will hold the data
        xor             IA0, IA0
%assign IDX 0
%assign K_IDX 1
%rep 16
        mov             IA1, [%%T_OUT + (IDX*PTR_SZ)]
        mov             DWORD(IA0), [%%T_MASK + (IDX*4)]
        kmovq           k %+ K_IDX, IA0
        vmovdqu8        [IA1]{k %+ K_IDX}, %%T %+ IDX
%assign IDX (IDX + 1)
%assign K_IDX (K_IDX + 1)
%if K_IDX > 7
%assign K_IDX 1                 ; iterate through K1 to K7
%endif
%endrep

%ifdef SAFE_DATA
        ;; Clear copied IV's
        vpxorq          %%T5, %%T5
        vmovdqu64       [%%T_IV + (0*64)], %%T5
        vmovdqu64       [%%T_IV + (1*64)], %%T5
%endif

%%_des_cfb_one_end:

%endmacro

;;; ===========================================================================
;;; Converts length into mask of DES blocks
;;; ===========================================================================
;;;
;;; MASK [out] - mask8 for value; for masked 64b loads and stores (r64)
;;; USES: IA0, IA1 IA2
;;; ASSUMES: SIZE - OFFSET < 64
%macro GET_MASK8 1
%define %%MASK %1

%ifidn IA1, rcx
%define myrcx IA1
%else
%define myrcx rcx
        mov     IA1, rcx
%endif
        mov     myrcx, SIZE
        sub     myrcx, OFFSET
        ;; - myrcx - remaining length
        ;; - divide by 8 (DES block size)
        ;; - create bit mask of the result
        mov     DWORD(%%MASK), 1
        shr     DWORD(myrcx), 3
        shl     DWORD(%%MASK), BYTE(myrcx)
        sub     DWORD(%%MASK), 1
%ifnidn IA1, rcx
        mov     rcx, IA1
%endif
%endmacro

;;; ===========================================================================
;;; DES CBC ENCRYPT CIPHER ONLY (1 to 8 DES blocks only)
;;; ===========================================================================
;;;
;;; NUM_DES_BLOCKS [in] - 1 to 8 DES blocks only
;;; DES_KS         [in] - pointer to transposed key schedule
;;;
;;; NOTE: clobbers OpMask registers
;;; REQUIRES: ZTMP0 - ZTMP13, ZW0-ZW15 (depends on NUM_DES_BLOCKS), ZIV0, ZIV1
%macro GEN_DES_ENC_CIPHER 2
%define %%NUM_DES_BLOCKS %1
%define %%DES_KS         %2

%assign RN 0
%assign LN 1
%assign RNN 2
%assign LNN 3
%rep %%NUM_DES_BLOCKS - 1
        DES_ENC_DEC     ENC, ZW %+ RN, ZW %+ LN, %%DES_KS, ZTMP0, ZTMP1, ZTMP2, ZTMP3, ZTMP4, ZTMP5, ZTMP6, ZTMP7, ZTMP8, ZTMP9, ZTMP10, ZTMP11
        vpxord          ZW %+ RNN, ZW %+ RNN, ZW %+ LN ; R1 = R1 ^ L0
        vpxord          ZW %+ LNN, ZW %+ LNN, ZW %+ RN ; L1 = L1 ^ R0
%assign RN (RN + 2)
%assign LN (LN + 2)
%assign RNN (RNN + 2)
%assign LNN (LNN + 2)
%endrep
        DES_ENC_DEC     ENC, ZW %+ RN, ZW %+ LN, %%DES_KS, ZTMP0, ZTMP1, ZTMP2, ZTMP3, ZTMP4, ZTMP5, ZTMP6, ZTMP7, ZTMP8, ZTMP9, ZTMP10, ZTMP11
        vmovdqa64       ZIV0, ZW %+ LN ; IV0 = L7
        vmovdqa64       ZIV1, ZW %+ RN ; IV1 = R7
%endmacro

;;; ===========================================================================
;;; DES CBC DECRYPT CIPHER ONLY (1 to 8 DES blocks only)
;;; ===========================================================================
;;;
;;; NUM_DES_BLOCKS [in] - 1 to 8 DES blocks only
;;; DES_KS         [in] - pointer to transposed key schedule
;;;
;;; NOTE: clobbers OpMask registers
;;; REQUIRES: ZTMP0 - ZTMP13, ZW0-ZW15 (depends on NUM_DES_BLOCKS), ZIV0, ZIV1
%macro GEN_DES_DEC_CIPHER 2
%define %%NUM_DES_BLOCKS %1
%define %%DES_KS         %2

%assign RN 0
%assign LN 1
%rep %%NUM_DES_BLOCKS
        vmovdqa64       ZTMP12, ZW %+ RN        ; keep R0 as IV for the next round
        vmovdqa64       ZTMP13, ZW %+ LN        ; keep L0 as IV for the next round
        DES_ENC_DEC     DEC, ZW %+ RN, ZW %+ LN, %%DES_KS, ZTMP0, ZTMP1, ZTMP2, ZTMP3, ZTMP4, ZTMP5, ZTMP6, ZTMP7, ZTMP8, ZTMP9, ZTMP10, ZTMP11
        vpxord          ZW %+ RN, ZW %+ RN, ZIV1 ; R0 = R0 ^ IV1
        vpxord          ZW %+ LN, ZW %+ LN, ZIV0 ; L0 = L0 ^ IV0
        vmovdqa64       ZIV0, ZTMP12
        vmovdqa64       ZIV1, ZTMP13
%assign RN (RN + 2)
%assign LN (LN + 2)
%endrep
%endmacro

;;; ===========================================================================
;;; 3DES CBC ENCRYPT CIPHER ONLY (1 to 8 DES blocks only)
;;; ===========================================================================
;;;
;;; NUM_DES_BLOCKS [in] - 1 to 8 DES blocks only
;;; DES_KS1        [in] - pointer to transposed key schedule 1
;;; DES_KS2        [in] - pointer to transposed key schedule 2
;;; DES_KS3        [in] - pointer to transposed key schedule 3
;;;
;;; NOTE: clobbers OpMask registers
;;; REQUIRES: ZTMP0 - ZTMP13, ZW0-ZW15 (depends on NUM_DES_BLOCKS), ZIV0, ZIV1
%macro GEN_3DES_ENC_CIPHER 4
%define %%NUM_DES_BLOCKS %1
%define %%DES_KS1        %2
%define %%DES_KS2        %3
%define %%DES_KS3        %4

%assign RN 0
%assign LN 1
%assign RNN 2
%assign LNN 3
%rep %%NUM_DES_BLOCKS
        ;; ENC
        DES_ENC_DEC     ENC, ZW %+ RN, ZW %+ LN, %%DES_KS1, ZTMP0, ZTMP1, ZTMP2, ZTMP3, ZTMP4, ZTMP5, ZTMP6, ZTMP7, ZTMP8, ZTMP9, ZTMP10, ZTMP11
        ;; DEC
        DES_ENC_DEC     DEC, ZW %+ LN, ZW %+ RN, %%DES_KS2, ZTMP0, ZTMP1, ZTMP2, ZTMP3, ZTMP4, ZTMP5, ZTMP6, ZTMP7, ZTMP8, ZTMP9, ZTMP10, ZTMP11
        ;; ENC
        DES_ENC_DEC     ENC, ZW %+ RN, ZW %+ LN, %%DES_KS3, ZTMP0, ZTMP1, ZTMP2, ZTMP3, ZTMP4, ZTMP5, ZTMP6, ZTMP7, ZTMP8, ZTMP9, ZTMP10, ZTMP11
%if (RNN < (%%NUM_DES_BLOCKS * 2))
        vpxord          ZW %+ RNN, ZW %+ RNN, ZW %+ LN ; R1 = R1 ^ L0
        vpxord          ZW %+ LNN, ZW %+ LNN, ZW %+ RN ; L1 = L1 ^ R0
%else
        vmovdqa64       ZIV0, ZW %+ LN ; IV0 = L7
        vmovdqa64       ZIV1, ZW %+ RN ; IV1 = R7
%endif

%assign RN (RN + 2)
%assign LN (LN + 2)
%assign RNN (RNN + 2)
%assign LNN (LNN + 2)
%endrep

%endmacro

;;; ===========================================================================
;;; 3DES CBC DECRYPT CIPHER ONLY (1 to 8 DES blocks only)
;;; ===========================================================================
;;;
;;; NUM_DES_BLOCKS [in] - 1 to 8 DES blocks only
;;; DES_KS1        [in] - pointer to transposed key schedule 1
;;; DES_KS2        [in] - pointer to transposed key schedule 2
;;; DES_KS3        [in] - pointer to transposed key schedule 3
;;;
;;; NOTE: clobbers OpMask registers
;;; REQUIRES: ZTMP0 - ZTMP13, ZW0-ZW15 (depends on NUM_DES_BLOCKS), ZIV0, ZIV1
%macro GEN_3DES_DEC_CIPHER 4
%define %%NUM_DES_BLOCKS %1
%define %%DES_KS1        %2
%define %%DES_KS2        %3
%define %%DES_KS3        %4

%assign RN 0
%assign LN 1
%rep %%NUM_DES_BLOCKS
        vmovdqa64       ZTMP12, ZW %+ RN        ; keep R0 as IV for the next round
        vmovdqa64       ZTMP13, ZW %+ LN        ; keep L0 as IV for the next round
        ;; DEC
        DES_ENC_DEC     DEC, ZW %+ RN, ZW %+ LN, %%DES_KS1, ZTMP0, ZTMP1, ZTMP2, ZTMP3, ZTMP4, ZTMP5, ZTMP6, ZTMP7, ZTMP8, ZTMP9, ZTMP10, ZTMP11
        ;; ENC
        DES_ENC_DEC     ENC, ZW %+ LN, ZW %+ RN, %%DES_KS2, ZTMP0, ZTMP1, ZTMP2, ZTMP3, ZTMP4, ZTMP5, ZTMP6, ZTMP7, ZTMP8, ZTMP9, ZTMP10, ZTMP11
        ;; DEC
        DES_ENC_DEC     DEC, ZW %+ RN, ZW %+ LN, %%DES_KS3, ZTMP0, ZTMP1, ZTMP2, ZTMP3, ZTMP4, ZTMP5, ZTMP6, ZTMP7, ZTMP8, ZTMP9, ZTMP10, ZTMP11
        vpxord          ZW %+ RN, ZW %+ RN, ZIV1 ; R0 = R0 ^ IV1
        vpxord          ZW %+ LN, ZW %+ LN, ZIV0 ; L0 = L0 ^ IV0
        vmovdqa64       ZIV0, ZTMP12
        vmovdqa64       ZIV1, ZTMP13

%assign RN (RN + 2)
%assign LN (LN + 2)
%endrep

%endmacro

;;; ===========================================================================
;;; DES CBC / DOCSIS DES ENCRYPT
;;; ===========================================================================
;;;
;;; DES_DOCSIS [in] - select between DES (DES CBC), DOCSIS (DOCSIS DES) and
;;;                   3DES (3DES CBC)
;;;
;;; NOTE: clobbers OpMask registers
%macro GENERIC_DES_ENC 1
%define %%DES_DOCSIS %1

        ;; push the registers and allocate the stack frame
	mov	        rax, rsp
	sub	        rsp, STACKFRAME_size
	and	        rsp, -64
	mov	        [rsp + _rsp_save], rax	; original SP
        mov             [rsp + _gpr_save + 0*8], r12
        mov             [rsp + _gpr_save + 1*8], r13
        mov             [rsp + _gpr_save + 2*8], r14
        mov             [rsp + _gpr_save + 3*8], r15

%ifnidn %%DES_DOCSIS, 3DES
        ;; DES and DOCSIS DES
        DES_INIT        STATE + _des_args_keys, STATE + _des_args_IV, rsp + _key_sched, ZIV0, ZIV1, ZW0, ZW1, ZW2, ZW3, ZW4, ZW5, ZW6, ZW7, ZW8, ZW9, ZW10, ZW11, ZW12, ZW13, ZW14, ZW15, ZTMP0, ZTMP1, ZTMP2, ZTMP3, ZTMP4, ZTMP5, ZTMP6, ZTMP7, ZTMP8, ZTMP9, ZTMP10, ZTMP11
%else
        ;; 3DES
        DES3_INIT        STATE + _des_args_keys, STATE + _des_args_IV, rsp + _key_sched, rsp + _key_sched2, rsp + _key_sched3, ZIV0, ZIV1, ZW0, ZW1, ZW2, ZW3, ZW4, ZW5, ZW6, ZW7, ZW8, ZW9, ZW10, ZW11, ZW12, ZW13, ZW14, ZW15, ZTMP0, ZTMP1, ZTMP2, ZTMP3, ZTMP4, ZTMP5, ZTMP6, ZTMP7, ZTMP8, ZTMP9, ZTMP10, ZTMP11, ENC
%endif
        mov             [rsp + _size_save], SIZE
        and             SIZE, -64
        xor             OFFSET, OFFSET
        ;; This loop processes message in blocks of 64 bytes.
        ;; Anything smaller than 64 bytes is handled separately after the loop.
%%_gen_des_enc_loop:
        cmp             OFFSET, SIZE
        jz              %%_gen_des_enc_loop_end
        ;; run loads
        mov             IA0, [STATE + _des_args_in + (0*PTR_SZ)]
        mov             IA1, [STATE + _des_args_in + (1*PTR_SZ)]
        mov             IA2, [STATE + _des_args_in + (2*PTR_SZ)]
        mov             INP0, [STATE + _des_args_in + (3*PTR_SZ)]
        mov             INP1, [STATE + _des_args_in + (4*PTR_SZ)]
        mov             INP2, [STATE + _des_args_in + (5*PTR_SZ)]
        mov             INP3, [STATE + _des_args_in + (6*PTR_SZ)]
        mov             INP4, [STATE + _des_args_in + (7*PTR_SZ)]
        vmovdqu64       ZW0, [IA0 + OFFSET]
        vmovdqu64       ZW1, [IA1 + OFFSET]
        vmovdqu64       ZW2, [IA2 + OFFSET]
        vmovdqu64       ZW3, [INP0 + OFFSET]
        vmovdqu64       ZW4, [INP1 + OFFSET]
        vmovdqu64       ZW5, [INP2 + OFFSET]
        vmovdqu64       ZW6, [INP3 + OFFSET]
        vmovdqu64       ZW7, [INP4 + OFFSET]

        mov             IA0, [STATE + _des_args_in + (8*PTR_SZ)]
        mov             IA1, [STATE + _des_args_in + (9*PTR_SZ)]
        mov             IA2, [STATE + _des_args_in + (10*PTR_SZ)]
        mov             INP0, [STATE + _des_args_in + (11*PTR_SZ)]
        mov             INP1, [STATE + _des_args_in + (12*PTR_SZ)]
        mov             INP2, [STATE + _des_args_in + (13*PTR_SZ)]
        mov             INP3, [STATE + _des_args_in + (14*PTR_SZ)]
        mov             INP4, [STATE + _des_args_in + (15*PTR_SZ)]
        vmovdqu64       ZW8, [IA0 + OFFSET]
        vmovdqu64       ZW9, [IA1 + OFFSET]
        vmovdqu64       ZW10, [IA2 + OFFSET]
        vmovdqu64       ZW11, [INP0 + OFFSET]
        vmovdqu64       ZW12, [INP1 + OFFSET]
        vmovdqu64       ZW13, [INP2 + OFFSET]
        vmovdqu64       ZW14, [INP3 + OFFSET]
        vmovdqu64       ZW15, [INP4 + OFFSET]

        ;; Transpose input
        TRANSPOSE_IN    ZW0, ZW1, ZW2, ZW3, ZW4, ZW5, ZW6, ZW7, ZW8, ZW9, ZW10, ZW11, ZW12, ZW13, ZW14, ZW15, ZTMP0, ZTMP1, ZTMP2, ZTMP3, ZTMP4, ZTMP5, ZTMP6, ZTMP7, ZTMP8, ZTMP9, ZTMP10, ZTMP11, ZTMP12, ZTMP13

        ;; DES CBC ENC comes here
        vpxord          ZW0, ZW0, ZIV0 ; R0 = R0 ^ IV0
        vpxord          ZW1, ZW1, ZIV1 ; L0 = L0 ^ IV1

%ifnidn %%DES_DOCSIS, 3DES
        GEN_DES_ENC_CIPHER 8, rsp + _key_sched
%else
        GEN_3DES_ENC_CIPHER 8, rsp + _key_sched, rsp + _key_sched2, rsp + _key_sched3
%endif

        ;; transpose data on output
        TRANSPOSE_OUT   ZW0, ZW1, ZW2, ZW3, ZW4, ZW5, ZW6, ZW7, ZW8, ZW9, ZW10, ZW11, ZW12, ZW13, ZW14, ZW15, ZTMP0, ZTMP1, ZTMP2, ZTMP3, ZTMP4, ZTMP5, ZTMP6, ZTMP7, ZTMP8, ZTMP9, ZTMP10, ZTMP11, ZTMP12, ZTMP13
        ;; run stores
        mov             IA0, [STATE + _des_args_out + (0*PTR_SZ)]
        mov             IA1, [STATE + _des_args_out + (1*PTR_SZ)]
        mov             IA2, [STATE + _des_args_out + (2*PTR_SZ)]
        mov             INP0, [STATE + _des_args_out + (3*PTR_SZ)]
        mov             INP1, [STATE + _des_args_out + (4*PTR_SZ)]
        mov             INP2, [STATE + _des_args_out + (5*PTR_SZ)]
        mov             INP3, [STATE + _des_args_out + (6*PTR_SZ)]
        mov             INP4, [STATE + _des_args_out + (7*PTR_SZ)]
        vmovdqu64       [IA0 + OFFSET], ZW0
        vmovdqu64       [IA1 + OFFSET], ZW1
        vmovdqu64       [IA2 + OFFSET], ZW2
        vmovdqu64       [INP0 + OFFSET], ZW3
        vmovdqu64       [INP1 + OFFSET], ZW4
        vmovdqu64       [INP2 + OFFSET], ZW5
        vmovdqu64       [INP3 + OFFSET], ZW6
        vmovdqu64       [INP4 + OFFSET], ZW7

        mov             IA0, [STATE + _des_args_out + (8*PTR_SZ)]
        mov             IA1, [STATE + _des_args_out + (9*PTR_SZ)]
        mov             IA2, [STATE + _des_args_out + (10*PTR_SZ)]
        mov             INP0, [STATE + _des_args_out + (11*PTR_SZ)]
        mov             INP1, [STATE + _des_args_out + (12*PTR_SZ)]
        mov             INP2, [STATE + _des_args_out + (13*PTR_SZ)]
        mov             INP3, [STATE + _des_args_out + (14*PTR_SZ)]
        mov             INP4, [STATE + _des_args_out + (15*PTR_SZ)]
        vmovdqu64       [IA0 + OFFSET], ZW8
        vmovdqu64       [IA1 + OFFSET], ZW9
        vmovdqu64       [IA2 + OFFSET], ZW10
        vmovdqu64       [INP0 + OFFSET], ZW11
        vmovdqu64       [INP1 + OFFSET], ZW12
        vmovdqu64       [INP2 + OFFSET], ZW13
        vmovdqu64       [INP3 + OFFSET], ZW14
        vmovdqu64       [INP4 + OFFSET], ZW15

        add             OFFSET, 64
        jmp             %%_gen_des_enc_loop
%%_gen_des_enc_loop_end:
        ;; This is where we check if there is anything less than 64 bytes
        ;; of message left for processing.
        mov             SIZE, [rsp + _size_save]
        cmp             OFFSET, SIZE
        jz              %%_gen_des_enc_part_end
        ;; calculate min of bytes_left and 64, convert to qword mask
        GET_MASK8       IA0        ; IA0 = mask

        kmovw           k7, DWORD(IA0)
        mov             [rsp + _mask_save], IA0
        ;; run masked loads
        mov             IA0, [STATE + _des_args_in + (0*PTR_SZ)]
        mov             IA1, [STATE + _des_args_in + (1*PTR_SZ)]
        mov             IA2, [STATE + _des_args_in + (2*PTR_SZ)]
        mov             INP0, [STATE + _des_args_in + (3*PTR_SZ)]
        mov             INP1, [STATE + _des_args_in + (4*PTR_SZ)]
        mov             INP2, [STATE + _des_args_in + (5*PTR_SZ)]
        mov             INP3, [STATE + _des_args_in + (6*PTR_SZ)]
        mov             INP4, [STATE + _des_args_in + (7*PTR_SZ)]
        vmovdqu64       ZW0{k7}{z}, [IA0 + OFFSET]
        vmovdqu64       ZW1{k7}{z}, [IA1 + OFFSET]
        vmovdqu64       ZW2{k7}{z}, [IA2 + OFFSET]
        vmovdqu64       ZW3{k7}{z}, [INP0 + OFFSET]
        vmovdqu64       ZW4{k7}{z}, [INP1 + OFFSET]
        vmovdqu64       ZW5{k7}{z}, [INP2 + OFFSET]
        vmovdqu64       ZW6{k7}{z}, [INP3 + OFFSET]
        vmovdqu64       ZW7{k7}{z}, [INP4 + OFFSET]

        mov             IA0, [STATE + _des_args_in + (8*PTR_SZ)]
        mov             IA1, [STATE + _des_args_in + (9*PTR_SZ)]
        mov             IA2, [STATE + _des_args_in + (10*PTR_SZ)]
        mov             INP0, [STATE + _des_args_in + (11*PTR_SZ)]
        mov             INP1, [STATE + _des_args_in + (12*PTR_SZ)]
        mov             INP2, [STATE + _des_args_in + (13*PTR_SZ)]
        mov             INP3, [STATE + _des_args_in + (14*PTR_SZ)]
        mov             INP4, [STATE + _des_args_in + (15*PTR_SZ)]
        vmovdqu64       ZW8{k7}{z}, [IA0 + OFFSET]
        vmovdqu64       ZW9{k7}{z}, [IA1 + OFFSET]
        vmovdqu64       ZW10{k7}{z}, [IA2 + OFFSET]
        vmovdqu64       ZW11{k7}{z}, [INP0 + OFFSET]
        vmovdqu64       ZW12{k7}{z}, [INP1 + OFFSET]
        vmovdqu64       ZW13{k7}{z}, [INP2 + OFFSET]
        vmovdqu64       ZW14{k7}{z}, [INP3 + OFFSET]
        vmovdqu64       ZW15{k7}{z}, [INP4 + OFFSET]

        ;; Transpose input
        TRANSPOSE_IN    ZW0, ZW1, ZW2, ZW3, ZW4, ZW5, ZW6, ZW7, ZW8, ZW9, ZW10, ZW11, ZW12, ZW13, ZW14, ZW15, ZTMP0, ZTMP1, ZTMP2, ZTMP3, ZTMP4, ZTMP5, ZTMP6, ZTMP7, ZTMP8, ZTMP9, ZTMP10, ZTMP11, ZTMP12, ZTMP13

        ;; DES CBC ENC comes here
        vpxord          ZW0, ZW0, ZIV0 ; R0 = R0 ^ IV0
        vpxord          ZW1, ZW1, ZIV1 ; L0 = L0 ^ IV1

        mov             IA0, [rsp + _mask_save]
        cmp             BYTE(IA0), 0x0f
        ja              %%_gt_4
        jz              %%_blocks_4

        cmp             BYTE(IA0), 0x03
        ja              %%_blocks_3
        jz              %%_blocks_2

        ;; process one block and move to transpose out
%ifnidn %%DES_DOCSIS, 3DES
        GEN_DES_ENC_CIPHER 1, rsp + _key_sched
%else
        GEN_3DES_ENC_CIPHER 1, rsp + _key_sched, rsp + _key_sched2, rsp + _key_sched3
%endif
        jmp             %%_transpose_out

%%_blocks_2:
        ;; process two blocks and move to transpose out
%ifnidn %%DES_DOCSIS, 3DES
        GEN_DES_ENC_CIPHER 2, rsp + _key_sched
%else
        GEN_3DES_ENC_CIPHER 2, rsp + _key_sched, rsp + _key_sched2, rsp + _key_sched3
%endif
        jmp             %%_transpose_out

%%_blocks_3:
        ;; process three blocks and move to transpose out
%ifnidn %%DES_DOCSIS, 3DES
        GEN_DES_ENC_CIPHER 3, rsp + _key_sched
%else
        GEN_3DES_ENC_CIPHER 3, rsp + _key_sched, rsp + _key_sched2, rsp + _key_sched3
%endif
        jmp             %%_transpose_out

%%_blocks_4:
        ;; process four blocks and move to transpose out
%ifnidn %%DES_DOCSIS, 3DES
        GEN_DES_ENC_CIPHER 4, rsp + _key_sched
%else
        GEN_3DES_ENC_CIPHER 4, rsp + _key_sched, rsp + _key_sched2, rsp + _key_sched3
%endif
        jmp             %%_transpose_out

%%_gt_4:
        cmp             BYTE(IA0), 0x3f
        ja              %%_blocks_7
        jz              %%_blocks_6
%%_blocks_5:
        ;; process five blocks and move to transpose out
%ifnidn %%DES_DOCSIS, 3DES
        GEN_DES_ENC_CIPHER 5, rsp + _key_sched
%else
        GEN_3DES_ENC_CIPHER 5, rsp + _key_sched, rsp + _key_sched2, rsp + _key_sched3
%endif
        jmp             %%_transpose_out

%%_blocks_6:
        ;; process six blocks and move to transpose out
%ifnidn %%DES_DOCSIS, 3DES
        GEN_DES_ENC_CIPHER 6, rsp + _key_sched
%else
        GEN_3DES_ENC_CIPHER 6, rsp + _key_sched, rsp + _key_sched2, rsp + _key_sched3
%endif
        jmp             %%_transpose_out

%%_blocks_7:
        ;; process seven blocks and move to transpose out
%ifnidn %%DES_DOCSIS, 3DES
        GEN_DES_ENC_CIPHER 7, rsp + _key_sched
%else
        GEN_3DES_ENC_CIPHER 7, rsp + _key_sched, rsp + _key_sched2, rsp + _key_sched3
%endif

%%_transpose_out:
        ;; transpose data on output
        TRANSPOSE_OUT   ZW0, ZW1, ZW2, ZW3, ZW4, ZW5, ZW6, ZW7, ZW8, ZW9, ZW10, ZW11, ZW12, ZW13, ZW14, ZW15, ZTMP0, ZTMP1, ZTMP2, ZTMP3, ZTMP4, ZTMP5, ZTMP6, ZTMP7, ZTMP8, ZTMP9, ZTMP10, ZTMP11, ZTMP12, ZTMP13

        ;; run masked stores
        mov             IA0, [STATE + _des_args_out + (0*PTR_SZ)]
        mov             IA1, [STATE + _des_args_out + (1*PTR_SZ)]
        mov             IA2, [STATE + _des_args_out + (2*PTR_SZ)]
        mov             INP0, [STATE + _des_args_out + (3*PTR_SZ)]
        mov             INP1, [STATE + _des_args_out + (4*PTR_SZ)]
        mov             INP2, [STATE + _des_args_out + (5*PTR_SZ)]
        mov             INP3, [STATE + _des_args_out + (6*PTR_SZ)]
        mov             INP4, [STATE + _des_args_out + (7*PTR_SZ)]
        vmovdqu64       [IA0 + OFFSET]{k7}, ZW0
        vmovdqu64       [IA1 + OFFSET]{k7}, ZW1
        vmovdqu64       [IA2 + OFFSET]{k7}, ZW2
        vmovdqu64       [INP0 + OFFSET]{k7}, ZW3
        vmovdqu64       [INP1 + OFFSET]{k7}, ZW4
        vmovdqu64       [INP2 + OFFSET]{k7}, ZW5
        vmovdqu64       [INP3 + OFFSET]{k7}, ZW6
        vmovdqu64       [INP4 + OFFSET]{k7}, ZW7

        mov             IA0, [STATE + _des_args_out + (8*PTR_SZ)]
        mov             IA1, [STATE + _des_args_out + (9*PTR_SZ)]
        mov             IA2, [STATE + _des_args_out + (10*PTR_SZ)]
        mov             INP0, [STATE + _des_args_out + (11*PTR_SZ)]
        mov             INP1, [STATE + _des_args_out + (12*PTR_SZ)]
        mov             INP2, [STATE + _des_args_out + (13*PTR_SZ)]
        mov             INP3, [STATE + _des_args_out + (14*PTR_SZ)]
        mov             INP4, [STATE + _des_args_out + (15*PTR_SZ)]
        vmovdqu64       [IA0 + OFFSET]{k7}, ZW8
        vmovdqu64       [IA1 + OFFSET]{k7}, ZW9
        vmovdqu64       [IA2 + OFFSET]{k7}, ZW10
        vmovdqu64       [INP0 + OFFSET]{k7}, ZW11
        vmovdqu64       [INP1 + OFFSET]{k7}, ZW12
        vmovdqu64       [INP2 + OFFSET]{k7}, ZW13
        vmovdqu64       [INP3 + OFFSET]{k7}, ZW14
        vmovdqu64       [INP4 + OFFSET]{k7}, ZW15
%%_gen_des_enc_part_end:

        ;; store IV and update pointers
        DES_FINISH      ZIV0, ZIV1, ZTMP0, ZTMP1, ZTMP2, ZTMP3, ZTMP4

        ;; CFB part for DOCSIS
%ifidn %%DES_DOCSIS, DOCSIS
        DES_CFB_ONE     ENC, rsp + _key_sched, ZW0, ZW1, ZW2, ZW3, ZW4, ZW5, ZW6, ZW7, ZW8, ZW9, ZW10, ZW11, ZW12, ZW13, ZW14, ZW15, ZTMP0, ZTMP1, ZTMP2, ZTMP3, ZTMP4, ZTMP5, ZTMP6, ZTMP7, ZTMP8, rsp + _tmp_in, rsp + _tmp_out, rsp + _tmp_iv, rsp + _tmp_mask
%endif

        CLEAR_KEY_SCHEDULE %%DES_DOCSIS, ZW0

        ;; restore stack pointer and registers
        mov             r12, [rsp + _gpr_save + 0*8]
        mov             r13, [rsp + _gpr_save + 1*8]
        mov             r14, [rsp + _gpr_save + 2*8]
        mov             r15, [rsp + _gpr_save + 3*8]
	mov	        rsp, [rsp + _rsp_save]	; original SP
%endmacro

;;; ===========================================================================
;;; DES CBC / DOCSIS DES DECRYPT
;;; ===========================================================================
;;;
;;; DES_DOCSIS [in] - select between DES (DES CBC), DOCSIS (DOCSIS DES) and
;;;                   3DES (3DES CBC)
;;;
;;; NOTE: clobbers OpMask registers
%macro GENERIC_DES_DEC 1
%define %%DES_DOCSIS %1

        ;; push the registers and allocate the stack frame
	mov	        rax, rsp
	sub	        rsp, STACKFRAME_size
	and	        rsp, -64
	mov	        [rsp + _rsp_save], rax	; original SP
        mov             [rsp + _gpr_save + 0*8], r12
        mov             [rsp + _gpr_save + 1*8], r13
        mov             [rsp + _gpr_save + 2*8], r14
        mov             [rsp + _gpr_save + 3*8], r15

%ifnidn %%DES_DOCSIS, 3DES
        ;; DES and DOCSIS
        DES_INIT        STATE + _des_args_keys, STATE + _des_args_IV, rsp + _key_sched, ZIV0, ZIV1, ZW0, ZW1, ZW2, ZW3, ZW4, ZW5, ZW6, ZW7, ZW8, ZW9, ZW10, ZW11, ZW12, ZW13, ZW14, ZW15, ZTMP0, ZTMP1, ZTMP2, ZTMP3, ZTMP4, ZTMP5, ZTMP6, ZTMP7, ZTMP8, ZTMP9, ZTMP10, ZTMP11
%else
        ;; 3DES
        DES3_INIT       STATE + _des_args_keys, STATE + _des_args_IV, rsp + _key_sched, rsp + _key_sched2, rsp + _key_sched3, ZIV0, ZIV1, ZW0, ZW1, ZW2, ZW3, ZW4, ZW5, ZW6, ZW7, ZW8, ZW9, ZW10, ZW11, ZW12, ZW13, ZW14, ZW15, ZTMP0, ZTMP1, ZTMP2, ZTMP3, ZTMP4, ZTMP5, ZTMP6, ZTMP7, ZTMP8, ZTMP9, ZTMP10, ZTMP11, DEC
%endif

        ;; CFB part for DOCSIS
%ifidn %%DES_DOCSIS, DOCSIS
        DES_CFB_ONE     DEC, rsp + _key_sched, ZW0, ZW1, ZW2, ZW3, ZW4, ZW5, ZW6, ZW7, ZW8, ZW9, ZW10, ZW11, ZW12, ZW13, ZW14, ZW15, ZTMP0, ZTMP1, ZTMP2, ZTMP3, ZTMP4, ZTMP5, ZTMP6, ZTMP7, ZTMP8, rsp + _tmp_in, rsp + _tmp_out, rsp + _tmp_iv, rsp + _tmp_mask
%endif

        mov             [rsp + _size_save], SIZE
        and             SIZE, -64
        xor             OFFSET, OFFSET
        ;; This loop processes message in blocks of 64 bytes.
        ;; Anything smaller than 64 bytes is handled separately after the loop.
%%_gen_des_dec_loop:
        cmp             OFFSET, SIZE
        jz              %%_gen_des_dec_loop_end
        ;; run loads
        mov             IA0, [STATE + _des_args_in + (0*PTR_SZ)]
        mov             IA1, [STATE + _des_args_in + (1*PTR_SZ)]
        mov             IA2, [STATE + _des_args_in + (2*PTR_SZ)]
        mov             INP0, [STATE + _des_args_in + (3*PTR_SZ)]
        mov             INP1, [STATE + _des_args_in + (4*PTR_SZ)]
        mov             INP2, [STATE + _des_args_in + (5*PTR_SZ)]
        mov             INP3, [STATE + _des_args_in + (6*PTR_SZ)]
        mov             INP4, [STATE + _des_args_in + (7*PTR_SZ)]
        vmovdqu64       ZW0, [IA0 + OFFSET]
        vmovdqu64       ZW1, [IA1 + OFFSET]
        vmovdqu64       ZW2, [IA2 + OFFSET]
        vmovdqu64       ZW3, [INP0 + OFFSET]
        vmovdqu64       ZW4, [INP1 + OFFSET]
        vmovdqu64       ZW5, [INP2 + OFFSET]
        vmovdqu64       ZW6, [INP3 + OFFSET]
        vmovdqu64       ZW7, [INP4 + OFFSET]

        mov             IA0, [STATE + _des_args_in + (8*PTR_SZ)]
        mov             IA1, [STATE + _des_args_in + (9*PTR_SZ)]
        mov             IA2, [STATE + _des_args_in + (10*PTR_SZ)]
        mov             INP0, [STATE + _des_args_in + (11*PTR_SZ)]
        mov             INP1, [STATE + _des_args_in + (12*PTR_SZ)]
        mov             INP2, [STATE + _des_args_in + (13*PTR_SZ)]
        mov             INP3, [STATE + _des_args_in + (14*PTR_SZ)]
        mov             INP4, [STATE + _des_args_in + (15*PTR_SZ)]
        vmovdqu64       ZW8, [IA0 + OFFSET]
        vmovdqu64       ZW9, [IA1 + OFFSET]
        vmovdqu64       ZW10, [IA2 + OFFSET]
        vmovdqu64       ZW11, [INP0 + OFFSET]
        vmovdqu64       ZW12, [INP1 + OFFSET]
        vmovdqu64       ZW13, [INP2 + OFFSET]
        vmovdqu64       ZW14, [INP3 + OFFSET]
        vmovdqu64       ZW15, [INP4 + OFFSET]

        ;; Transpose input
        TRANSPOSE_IN   ZW0, ZW1, ZW2, ZW3, ZW4, ZW5, ZW6, ZW7, ZW8, ZW9, ZW10, ZW11, ZW12, ZW13, ZW14, ZW15, ZTMP0, ZTMP1, ZTMP2, ZTMP3, ZTMP4, ZTMP5, ZTMP6, ZTMP7, ZTMP8, ZTMP9, ZTMP10, ZTMP11, ZTMP12, ZTMP13

%ifnidn %%DES_DOCSIS, 3DES
        ;; DES CBC DEC comes here
        GEN_DES_DEC_CIPHER 8, rsp + _key_sched
%else
        ;; 3DES CBC DEC comes here
        GEN_3DES_DEC_CIPHER 8, rsp + _key_sched, rsp + _key_sched2, rsp + _key_sched3
%endif

        ;; transpose data on output
        TRANSPOSE_OUT   ZW0, ZW1, ZW2, ZW3, ZW4, ZW5, ZW6, ZW7, ZW8, ZW9, ZW10, ZW11, ZW12, ZW13, ZW14, ZW15, ZTMP0, ZTMP1, ZTMP2, ZTMP3, ZTMP4, ZTMP5, ZTMP6, ZTMP7, ZTMP8, ZTMP9, ZTMP10, ZTMP11, ZTMP12, ZTMP13

        ;; run stores
        mov             IA0, [STATE + _des_args_out + (0*PTR_SZ)]
        mov             IA1, [STATE + _des_args_out + (1*PTR_SZ)]
        mov             IA2, [STATE + _des_args_out + (2*PTR_SZ)]
        mov             INP0, [STATE + _des_args_out + (3*PTR_SZ)]
        mov             INP1, [STATE + _des_args_out + (4*PTR_SZ)]
        mov             INP2, [STATE + _des_args_out + (5*PTR_SZ)]
        mov             INP3, [STATE + _des_args_out + (6*PTR_SZ)]
        mov             INP4, [STATE + _des_args_out + (7*PTR_SZ)]
        vmovdqu64       [IA0 + OFFSET], ZW0
        vmovdqu64       [IA1 + OFFSET], ZW1
        vmovdqu64       [IA2 + OFFSET], ZW2
        vmovdqu64       [INP0 + OFFSET], ZW3
        vmovdqu64       [INP1 + OFFSET], ZW4
        vmovdqu64       [INP2 + OFFSET], ZW5
        vmovdqu64       [INP3 + OFFSET], ZW6
        vmovdqu64       [INP4 + OFFSET], ZW7

        mov             IA0, [STATE + _des_args_out + (8*PTR_SZ)]
        mov             IA1, [STATE + _des_args_out + (9*PTR_SZ)]
        mov             IA2, [STATE + _des_args_out + (10*PTR_SZ)]
        mov             INP0, [STATE + _des_args_out + (11*PTR_SZ)]
        mov             INP1, [STATE + _des_args_out + (12*PTR_SZ)]
        mov             INP2, [STATE + _des_args_out + (13*PTR_SZ)]
        mov             INP3, [STATE + _des_args_out + (14*PTR_SZ)]
        mov             INP4, [STATE + _des_args_out + (15*PTR_SZ)]
        vmovdqu64       [IA0 + OFFSET], ZW8
        vmovdqu64       [IA1 + OFFSET], ZW9
        vmovdqu64       [IA2 + OFFSET], ZW10
        vmovdqu64       [INP0 + OFFSET], ZW11
        vmovdqu64       [INP1 + OFFSET], ZW12
        vmovdqu64       [INP2 + OFFSET], ZW13
        vmovdqu64       [INP3 + OFFSET], ZW14
        vmovdqu64       [INP4 + OFFSET], ZW15

        add             OFFSET, 64
        jmp             %%_gen_des_dec_loop
%%_gen_des_dec_loop_end:
        ;; This is where we check if there is anything less than 64 bytes
        ;; of message left for processing.
        mov             SIZE, [rsp + _size_save]
        cmp             OFFSET, SIZE
        jz              %%_gen_des_dec_part_end
        ;; calculate min of bytes_left and 64, convert to qword mask
        GET_MASK8       IA0        ; IA0 = mask

        kmovw           k7, DWORD(IA0)
        mov             [rsp + _mask_save], IA0
        ;; run masked loads
        mov             IA0, [STATE + _des_args_in + (0*PTR_SZ)]
        mov             IA1, [STATE + _des_args_in + (1*PTR_SZ)]
        mov             IA2, [STATE + _des_args_in + (2*PTR_SZ)]
        mov             INP0, [STATE + _des_args_in + (3*PTR_SZ)]
        mov             INP1, [STATE + _des_args_in + (4*PTR_SZ)]
        mov             INP2, [STATE + _des_args_in + (5*PTR_SZ)]
        mov             INP3, [STATE + _des_args_in + (6*PTR_SZ)]
        mov             INP4, [STATE + _des_args_in + (7*PTR_SZ)]
        vmovdqu64       ZW0{k7}{z}, [IA0 + OFFSET]
        vmovdqu64       ZW1{k7}{z}, [IA1 + OFFSET]
        vmovdqu64       ZW2{k7}{z}, [IA2 + OFFSET]
        vmovdqu64       ZW3{k7}{z}, [INP0 + OFFSET]
        vmovdqu64       ZW4{k7}{z}, [INP1 + OFFSET]
        vmovdqu64       ZW5{k7}{z}, [INP2 + OFFSET]
        vmovdqu64       ZW6{k7}{z}, [INP3 + OFFSET]
        vmovdqu64       ZW7{k7}{z}, [INP4 + OFFSET]

        mov             IA0, [STATE + _des_args_in + (8*PTR_SZ)]
        mov             IA1, [STATE + _des_args_in + (9*PTR_SZ)]
        mov             IA2, [STATE + _des_args_in + (10*PTR_SZ)]
        mov             INP0, [STATE + _des_args_in + (11*PTR_SZ)]
        mov             INP1, [STATE + _des_args_in + (12*PTR_SZ)]
        mov             INP2, [STATE + _des_args_in + (13*PTR_SZ)]
        mov             INP3, [STATE + _des_args_in + (14*PTR_SZ)]
        mov             INP4, [STATE + _des_args_in + (15*PTR_SZ)]
        vmovdqu64       ZW8{k7}{z}, [IA0 + OFFSET]
        vmovdqu64       ZW9{k7}{z}, [IA1 + OFFSET]
        vmovdqu64       ZW10{k7}{z}, [IA2 + OFFSET]
        vmovdqu64       ZW11{k7}{z}, [INP0 + OFFSET]
        vmovdqu64       ZW12{k7}{z}, [INP1 + OFFSET]
        vmovdqu64       ZW13{k7}{z}, [INP2 + OFFSET]
        vmovdqu64       ZW14{k7}{z}, [INP3 + OFFSET]
        vmovdqu64       ZW15{k7}{z}, [INP4 + OFFSET]

        ;; Transpose input
        TRANSPOSE_IN    ZW0, ZW1, ZW2, ZW3, ZW4, ZW5, ZW6, ZW7, ZW8, ZW9, ZW10, ZW11, ZW12, ZW13, ZW14, ZW15, ZTMP0, ZTMP1, ZTMP2, ZTMP3, ZTMP4, ZTMP5, ZTMP6, ZTMP7, ZTMP8, ZTMP9, ZTMP10, ZTMP11, ZTMP12, ZTMP13

        ;; DES CBC DEC comes here
        mov             IA0, [rsp + _mask_save]
        cmp             BYTE(IA0), 0x0f
        ja              %%_gt_4
        jz              %%_blocks_4

        cmp             BYTE(IA0), 0x03
        ja              %%_blocks_3
        jz              %%_blocks_2
        ;; process one block and move to transpose out
%ifnidn %%DES_DOCSIS, 3DES
        GEN_DES_DEC_CIPHER 1, rsp + _key_sched
%else
        GEN_3DES_DEC_CIPHER 1, rsp + _key_sched, rsp + _key_sched2, rsp + _key_sched3
%endif
        jmp             %%_transpose_out

%%_blocks_2:
        ;; process two blocks and move to transpose out
%ifnidn %%DES_DOCSIS, 3DES
        GEN_DES_DEC_CIPHER 2, rsp + _key_sched
%else
        GEN_3DES_DEC_CIPHER 2, rsp + _key_sched, rsp + _key_sched2, rsp + _key_sched3
%endif
        jmp             %%_transpose_out

%%_blocks_3:
        ;; process three blocks and move to transpose out
%ifnidn %%DES_DOCSIS, 3DES
        GEN_DES_DEC_CIPHER 3, rsp + _key_sched
%else
        GEN_3DES_DEC_CIPHER 3, rsp + _key_sched, rsp + _key_sched2, rsp + _key_sched3
%endif
        jmp             %%_transpose_out

%%_blocks_4:
        ;; process four blocks and move to transpose out
%ifnidn %%DES_DOCSIS, 3DES
        GEN_DES_DEC_CIPHER 4, rsp + _key_sched
%else
        GEN_3DES_DEC_CIPHER 4, rsp + _key_sched, rsp + _key_sched2, rsp + _key_sched3
%endif
        jmp             %%_transpose_out

%%_gt_4:
        cmp             BYTE(IA0), 0x3f
        ja              %%_blocks_7
        jz              %%_blocks_6
%%_blocks_5:
        ;; process five blocks and move to transpose out
%ifnidn %%DES_DOCSIS, 3DES
        GEN_DES_DEC_CIPHER 5, rsp + _key_sched
%else
        GEN_3DES_DEC_CIPHER 5, rsp + _key_sched, rsp + _key_sched2, rsp + _key_sched3
%endif
        jmp             %%_transpose_out

%%_blocks_6:
        ;; process six blocks and move to transpose out
%ifnidn %%DES_DOCSIS, 3DES
        GEN_DES_DEC_CIPHER 6, rsp + _key_sched
%else
        GEN_3DES_DEC_CIPHER 6, rsp + _key_sched, rsp + _key_sched2, rsp + _key_sched3
%endif
        jmp             %%_transpose_out

%%_blocks_7:
        ;; process seven blocks and move to transpose out
%ifnidn %%DES_DOCSIS, 3DES
        GEN_DES_DEC_CIPHER 7, rsp + _key_sched
%else
        GEN_3DES_DEC_CIPHER 7, rsp + _key_sched, rsp + _key_sched2, rsp + _key_sched3
%endif

%%_transpose_out:
        ;; transpose data on output
        TRANSPOSE_OUT   ZW0, ZW1, ZW2, ZW3, ZW4, ZW5, ZW6, ZW7, ZW8, ZW9, ZW10, ZW11, ZW12, ZW13, ZW14, ZW15, ZTMP0, ZTMP1, ZTMP2, ZTMP3, ZTMP4, ZTMP5, ZTMP6, ZTMP7, ZTMP8, ZTMP9, ZTMP10, ZTMP11, ZTMP12, ZTMP13

        ;; run masked stores
        mov             IA0, [STATE + _des_args_out + (0*PTR_SZ)]
        mov             IA1, [STATE + _des_args_out + (1*PTR_SZ)]
        mov             IA2, [STATE + _des_args_out + (2*PTR_SZ)]
        mov             INP0, [STATE + _des_args_out + (3*PTR_SZ)]
        mov             INP1, [STATE + _des_args_out + (4*PTR_SZ)]
        mov             INP2, [STATE + _des_args_out + (5*PTR_SZ)]
        mov             INP3, [STATE + _des_args_out + (6*PTR_SZ)]
        mov             INP4, [STATE + _des_args_out + (7*PTR_SZ)]
        vmovdqu64       [IA0 + OFFSET]{k7}, ZW0
        vmovdqu64       [IA1 + OFFSET]{k7}, ZW1
        vmovdqu64       [IA2 + OFFSET]{k7}, ZW2
        vmovdqu64       [INP0 + OFFSET]{k7}, ZW3
        vmovdqu64       [INP1 + OFFSET]{k7}, ZW4
        vmovdqu64       [INP2 + OFFSET]{k7}, ZW5
        vmovdqu64       [INP3 + OFFSET]{k7}, ZW6
        vmovdqu64       [INP4 + OFFSET]{k7}, ZW7

        mov             IA0, [STATE + _des_args_out + (8*PTR_SZ)]
        mov             IA1, [STATE + _des_args_out + (9*PTR_SZ)]
        mov             IA2, [STATE + _des_args_out + (10*PTR_SZ)]
        mov             INP0, [STATE + _des_args_out + (11*PTR_SZ)]
        mov             INP1, [STATE + _des_args_out + (12*PTR_SZ)]
        mov             INP2, [STATE + _des_args_out + (13*PTR_SZ)]
        mov             INP3, [STATE + _des_args_out + (14*PTR_SZ)]
        mov             INP4, [STATE + _des_args_out + (15*PTR_SZ)]
        vmovdqu64       [IA0 + OFFSET]{k7}, ZW8
        vmovdqu64       [IA1 + OFFSET]{k7}, ZW9
        vmovdqu64       [IA2 + OFFSET]{k7}, ZW10
        vmovdqu64       [INP0 + OFFSET]{k7}, ZW11
        vmovdqu64       [INP1 + OFFSET]{k7}, ZW12
        vmovdqu64       [INP2 + OFFSET]{k7}, ZW13
        vmovdqu64       [INP3 + OFFSET]{k7}, ZW14
        vmovdqu64       [INP4 + OFFSET]{k7}, ZW15
%%_gen_des_dec_part_end:

        ;; store IV and update pointers
        DES_FINISH      ZIV0, ZIV1, ZTMP0, ZTMP1, ZTMP2, ZTMP3, ZTMP4

        CLEAR_KEY_SCHEDULE %%DES_DOCSIS, ZW0

        ;; restore stack pointer and registers
        mov             r12, [rsp + _gpr_save + 0*8]
        mov             r13, [rsp + _gpr_save + 1*8]
        mov             r14, [rsp + _gpr_save + 2*8]
        mov             r15, [rsp + _gpr_save + 3*8]
	mov	        rsp, [rsp + _rsp_save]	; original SP
%endmacro


;;; ========================================================
;;; DATA

section .data
default rel
align 64
mask_values:
        dd 0x04000000, 0x04000000, 0x04000000, 0x04000000
        dd 0x04000000, 0x04000000, 0x04000000, 0x04000000
        dd 0x04000000, 0x04000000, 0x04000000, 0x04000000
        dd 0x04000000, 0x04000000, 0x04000000, 0x04000000
        dd 0x40240202, 0x40240202, 0x40240202, 0x40240202
        dd 0x40240202, 0x40240202, 0x40240202, 0x40240202
        dd 0x40240202, 0x40240202, 0x40240202, 0x40240202
        dd 0x40240202, 0x40240202, 0x40240202, 0x40240202
        dd 0x00001110, 0x00001110, 0x00001110, 0x00001110
        dd 0x00001110, 0x00001110, 0x00001110, 0x00001110
        dd 0x00001110, 0x00001110, 0x00001110, 0x00001110
        dd 0x00001110, 0x00001110, 0x00001110, 0x00001110
        dd 0x01088000, 0x01088000, 0x01088000, 0x01088000
        dd 0x01088000, 0x01088000, 0x01088000, 0x01088000
        dd 0x01088000, 0x01088000, 0x01088000, 0x01088000
        dd 0x01088000, 0x01088000, 0x01088000, 0x01088000
        dd 0x00000001, 0x00000001, 0x00000001, 0x00000001
        dd 0x00000001, 0x00000001, 0x00000001, 0x00000001
        dd 0x00000001, 0x00000001, 0x00000001, 0x00000001
        dd 0x00000001, 0x00000001, 0x00000001, 0x00000001
        dd 0x0081000C, 0x0081000C, 0x0081000C, 0x0081000C
        dd 0x0081000C, 0x0081000C, 0x0081000C, 0x0081000C
        dd 0x0081000C, 0x0081000C, 0x0081000C, 0x0081000C
        dd 0x0081000C, 0x0081000C, 0x0081000C, 0x0081000C
        dd 0x00000020, 0x00000020, 0x00000020, 0x00000020
        dd 0x00000020, 0x00000020, 0x00000020, 0x00000020
        dd 0x00000020, 0x00000020, 0x00000020, 0x00000020
        dd 0x00000020, 0x00000020, 0x00000020, 0x00000020
        dd 0x00000040, 0x00000040, 0x00000040, 0x00000040
        dd 0x00000040, 0x00000040, 0x00000040, 0x00000040
        dd 0x00000040, 0x00000040, 0x00000040, 0x00000040
        dd 0x00000040, 0x00000040, 0x00000040, 0x00000040
        dd 0x00400400, 0x00400400, 0x00400400, 0x00400400
        dd 0x00400400, 0x00400400, 0x00400400, 0x00400400
        dd 0x00400400, 0x00400400, 0x00400400, 0x00400400
        dd 0x00400400, 0x00400400, 0x00400400, 0x00400400
        dd 0x00000800, 0x00000800, 0x00000800, 0x00000800
        dd 0x00000800, 0x00000800, 0x00000800, 0x00000800
        dd 0x00000800, 0x00000800, 0x00000800, 0x00000800
        dd 0x00000800, 0x00000800, 0x00000800, 0x00000800
        dd 0x00002000, 0x00002000, 0x00002000, 0x00002000
        dd 0x00002000, 0x00002000, 0x00002000, 0x00002000
        dd 0x00002000, 0x00002000, 0x00002000, 0x00002000
        dd 0x00002000, 0x00002000, 0x00002000, 0x00002000
        dd 0x00100000, 0x00100000, 0x00100000, 0x00100000
        dd 0x00100000, 0x00100000, 0x00100000, 0x00100000
        dd 0x00100000, 0x00100000, 0x00100000, 0x00100000
        dd 0x00100000, 0x00100000, 0x00100000, 0x00100000
        dd 0x00004000, 0x00004000, 0x00004000, 0x00004000
        dd 0x00004000, 0x00004000, 0x00004000, 0x00004000
        dd 0x00004000, 0x00004000, 0x00004000, 0x00004000
        dd 0x00004000, 0x00004000, 0x00004000, 0x00004000
        dd 0x00020000, 0x00020000, 0x00020000, 0x00020000
        dd 0x00020000, 0x00020000, 0x00020000, 0x00020000
        dd 0x00020000, 0x00020000, 0x00020000, 0x00020000
        dd 0x00020000, 0x00020000, 0x00020000, 0x00020000
        dd 0x02000000, 0x02000000, 0x02000000, 0x02000000
        dd 0x02000000, 0x02000000, 0x02000000, 0x02000000
        dd 0x02000000, 0x02000000, 0x02000000, 0x02000000
        dd 0x02000000, 0x02000000, 0x02000000, 0x02000000
        dd 0x08000000, 0x08000000, 0x08000000, 0x08000000
        dd 0x08000000, 0x08000000, 0x08000000, 0x08000000
        dd 0x08000000, 0x08000000, 0x08000000, 0x08000000
        dd 0x08000000, 0x08000000, 0x08000000, 0x08000000
        dd 0x00000080, 0x00000080, 0x00000080, 0x00000080
        dd 0x00000080, 0x00000080, 0x00000080, 0x00000080
        dd 0x00000080, 0x00000080, 0x00000080, 0x00000080
        dd 0x00000080, 0x00000080, 0x00000080, 0x00000080
        dd 0x20000000, 0x20000000, 0x20000000, 0x20000000
        dd 0x20000000, 0x20000000, 0x20000000, 0x20000000
        dd 0x20000000, 0x20000000, 0x20000000, 0x20000000
        dd 0x20000000, 0x20000000, 0x20000000, 0x20000000
        dd 0x90000000, 0x90000000, 0x90000000, 0x90000000
        dd 0x90000000, 0x90000000, 0x90000000, 0x90000000
        dd 0x90000000, 0x90000000, 0x90000000, 0x90000000
        dd 0x90000000, 0x90000000, 0x90000000, 0x90000000

align 64
init_perm_consts:
        dd 0x0f0f0f0f, 0x0f0f0f0f, 0x0f0f0f0f, 0x0f0f0f0f
        dd 0x0f0f0f0f, 0x0f0f0f0f, 0x0f0f0f0f, 0x0f0f0f0f
        dd 0x0f0f0f0f, 0x0f0f0f0f, 0x0f0f0f0f, 0x0f0f0f0f
        dd 0x0f0f0f0f, 0x0f0f0f0f, 0x0f0f0f0f, 0x0f0f0f0f
        dd 0x0000ffff, 0x0000ffff, 0x0000ffff, 0x0000ffff
        dd 0x0000ffff, 0x0000ffff, 0x0000ffff, 0x0000ffff
        dd 0x0000ffff, 0x0000ffff, 0x0000ffff, 0x0000ffff
        dd 0x0000ffff, 0x0000ffff, 0x0000ffff, 0x0000ffff
        dd 0x33333333, 0x33333333, 0x33333333, 0x33333333
        dd 0x33333333, 0x33333333, 0x33333333, 0x33333333
        dd 0x33333333, 0x33333333, 0x33333333, 0x33333333
        dd 0x33333333, 0x33333333, 0x33333333, 0x33333333
        dd 0x00ff00ff, 0x00ff00ff, 0x00ff00ff, 0x00ff00ff
        dd 0x00ff00ff, 0x00ff00ff, 0x00ff00ff, 0x00ff00ff
        dd 0x00ff00ff, 0x00ff00ff, 0x00ff00ff, 0x00ff00ff
        dd 0x00ff00ff, 0x00ff00ff, 0x00ff00ff, 0x00ff00ff
        dd 0x55555555, 0x55555555, 0x55555555, 0x55555555
        dd 0x55555555, 0x55555555, 0x55555555, 0x55555555
        dd 0x55555555, 0x55555555, 0x55555555, 0x55555555
        dd 0x55555555, 0x55555555, 0x55555555, 0x55555555

;;; S-Box table
align 64
S_box_flipped:
        ;; SBOX0
        dw 0x07, 0x02, 0x0c, 0x0f, 0x04, 0x0b, 0x0a, 0x0c
        dw 0x0b, 0x07, 0x06, 0x09, 0x0d, 0x04, 0x00, 0x0a
        dw 0x02, 0x08, 0x05, 0x03, 0x0f, 0x06, 0x09, 0x05
        dw 0x08, 0x01, 0x03, 0x0e, 0x01, 0x0d, 0x0e, 0x00
        dw 0x00, 0x0f, 0x05, 0x0a, 0x07, 0x02, 0x09, 0x05
        dw 0x0e, 0x01, 0x03, 0x0c, 0x0b, 0x08, 0x0c, 0x06
        dw 0x0f, 0x03, 0x06, 0x0d, 0x04, 0x09, 0x0a, 0x00
        dw 0x02, 0x04, 0x0d, 0x07, 0x08, 0x0e, 0x01, 0x0b
        ;; SBOX1
        dw 0x0f, 0x00, 0x09, 0x0a, 0x06, 0x05, 0x03, 0x09
        dw 0x01, 0x0e, 0x04, 0x03, 0x0c, 0x0b, 0x0a, 0x04
        dw 0x08, 0x07, 0x0e, 0x01, 0x0d, 0x02, 0x00, 0x0c
        dw 0x07, 0x0d, 0x0b, 0x06, 0x02, 0x08, 0x05, 0x0f
        dw 0x0c, 0x0b, 0x03, 0x0d, 0x0f, 0x0c, 0x06, 0x00
        dw 0x02, 0x05, 0x08, 0x0e, 0x01, 0x02, 0x0d, 0x07
        dw 0x0b, 0x01, 0x00, 0x06, 0x04, 0x0f, 0x09, 0x0a
        dw 0x0e, 0x08, 0x05, 0x03, 0x07, 0x04, 0x0a, 0x09
        ;; SBOX2
        dw 0x05, 0x0b, 0x08, 0x0d, 0x06, 0x01, 0x0d, 0x0a
        dw 0x09, 0x02, 0x03, 0x04, 0x0f, 0x0c, 0x04, 0x07
        dw 0x00, 0x06, 0x0b, 0x08, 0x0c, 0x0f, 0x02, 0x05
        dw 0x07, 0x09, 0x0e, 0x03, 0x0a, 0x00, 0x01, 0x0e
        dw 0x0b, 0x08, 0x04, 0x02, 0x0c, 0x06, 0x03, 0x0d
        dw 0x00, 0x0b, 0x0a, 0x07, 0x06, 0x01, 0x0f, 0x04
        dw 0x0e, 0x05, 0x01, 0x0f, 0x02, 0x09, 0x0d, 0x0a
        dw 0x09, 0x00, 0x07, 0x0c, 0x05, 0x0e, 0x08, 0x03
        ;; SBOX3
        dw 0x0e, 0x05, 0x08, 0x0f, 0x00, 0x03, 0x0d, 0x0a
        dw 0x07, 0x09, 0x01, 0x0c, 0x09, 0x0e, 0x02, 0x01
        dw 0x0b, 0x06, 0x04, 0x08, 0x06, 0x0d, 0x03, 0x04
        dw 0x0c, 0x00, 0x0a, 0x07, 0x05, 0x0b, 0x0f, 0x02
        dw 0x0b, 0x0c, 0x02, 0x09, 0x06, 0x05, 0x08, 0x03
        dw 0x0d, 0x00, 0x04, 0x0a, 0x00, 0x0b, 0x07, 0x04
        dw 0x01, 0x0f, 0x0e, 0x02, 0x0f, 0x08, 0x05, 0x0e
        dw 0x0a, 0x06, 0x03, 0x0d, 0x0c, 0x01, 0x09, 0x07
        ;; SBOX4
        dw 0x04, 0x02, 0x01, 0x0f, 0x0e, 0x05, 0x0b, 0x06
        dw 0x02, 0x08, 0x0c, 0x03, 0x0d, 0x0e, 0x07, 0x00
        dw 0x03, 0x04, 0x0a, 0x09, 0x05, 0x0b, 0x00, 0x0c
        dw 0x08, 0x0d, 0x0f, 0x0a, 0x06, 0x01, 0x09, 0x07
        dw 0x07, 0x0d, 0x0a, 0x06, 0x02, 0x08, 0x0c, 0x05
        dw 0x04, 0x03, 0x0f, 0x00, 0x0b, 0x04, 0x01, 0x0a
        dw 0x0d, 0x01, 0x00, 0x0f, 0x0e, 0x07, 0x09, 0x02
        dw 0x03, 0x0e, 0x05, 0x09, 0x08, 0x0b, 0x06, 0x0c
        ;; SBOX5
        dw 0x03, 0x09, 0x00, 0x0e, 0x09, 0x04, 0x07, 0x08
        dw 0x05, 0x0f, 0x0c, 0x02, 0x06, 0x03, 0x0a, 0x0d
        dw 0x08, 0x07, 0x0b, 0x00, 0x04, 0x01, 0x0e, 0x0b
        dw 0x0f, 0x0a, 0x02, 0x05, 0x01, 0x0c, 0x0d, 0x06
        dw 0x05, 0x02, 0x06, 0x0d, 0x0e, 0x09, 0x00, 0x06
        dw 0x02, 0x04, 0x0b, 0x08, 0x09, 0x0f, 0x0c, 0x01
        dw 0x0f, 0x0c, 0x08, 0x07, 0x03, 0x0a, 0x0d, 0x00
        dw 0x04, 0x03, 0x07, 0x0e, 0x0a, 0x05, 0x01, 0x0b
        ;; SBOX6
        dw 0x02, 0x08, 0x0c, 0x05, 0x0f, 0x03, 0x0a, 0x00
        dw 0x04, 0x0d, 0x09, 0x06, 0x01, 0x0e, 0x06, 0x09
        dw 0x0d, 0x02, 0x03, 0x0f, 0x00, 0x0c, 0x05, 0x0a
        dw 0x07, 0x0b, 0x0e, 0x01, 0x0b, 0x07, 0x08, 0x04
        dw 0x0b, 0x06, 0x07, 0x09, 0x02, 0x08, 0x04, 0x07
        dw 0x0d, 0x0b, 0x0a, 0x00, 0x08, 0x05, 0x01, 0x0c
        dw 0x00, 0x0d, 0x0c, 0x0a, 0x09, 0x02, 0x0f, 0x04
        dw 0x0e, 0x01, 0x03, 0x0f, 0x05, 0x0e, 0x06, 0x03
        ;; SBOX7
        dw 0x0b, 0x0e, 0x05, 0x00, 0x06, 0x09, 0x0a, 0x0f
        dw 0x01, 0x02, 0x0c, 0x05, 0x0d, 0x07, 0x03, 0x0a
        dw 0x04, 0x0d, 0x09, 0x06, 0x0f, 0x03, 0x00, 0x0c
        dw 0x02, 0x08, 0x07, 0x0b, 0x08, 0x04, 0x0e, 0x01
        dw 0x08, 0x04, 0x03, 0x0f, 0x05, 0x02, 0x00, 0x0c
        dw 0x0b, 0x07, 0x06, 0x09, 0x0e, 0x01, 0x09, 0x06
        dw 0x0f, 0x08, 0x0a, 0x03, 0x0c, 0x05, 0x07, 0x0a
        dw 0x01, 0x0e, 0x0d, 0x00, 0x02, 0x0b, 0x04, 0x0d

;;; Used in DOCSIS DES partial block scheduling 16 x 32bit of value 1
align 64
vec_ones_32b:
        dd 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1

align 64
and_eu:
        dd 0x3f003f00, 0x3f003f00, 0x3f003f00, 0x3f003f00
        dd 0x3f003f00, 0x3f003f00, 0x3f003f00, 0x3f003f00
        dd 0x3f003f00, 0x3f003f00, 0x3f003f00, 0x3f003f00
        dd 0x3f003f00, 0x3f003f00, 0x3f003f00, 0x3f003f00

align 64
and_ed:
        dd 0x003f003f, 0x003f003f, 0x003f003f, 0x003f003f
        dd 0x003f003f, 0x003f003f, 0x003f003f, 0x003f003f
        dd 0x003f003f, 0x003f003f, 0x003f003f, 0x003f003f
        dd 0x003f003f, 0x003f003f, 0x003f003f, 0x003f003f

align 64
idx_e:
        dq 0x0d0c090805040100, 0x0f0e0b0a07060302
        dq 0x1d1c191815141110, 0x1f1e1b1a17161312
        dq 0x2d2c292825242120, 0x2f2e2b2a27262322
        dq 0x3d3c393835343130, 0x3f3e3b3a37363332

align 64
reg_values16bit_7:
        dq 0x001f001f001f001f, 0x001f001f001f001f
        dq 0x001f001f001f001f, 0x001f001f001f001f
        dq 0x001f001f001f001f, 0x001f001f001f001f
        dq 0x001f001f001f001f, 0x001f001f001f001f

align 64
shuffle_reg:
        dq 0x0705060403010200, 0x0f0d0e0c0b090a08
        dq 0x1715161413111210, 0x1f1d1e1c1b191a18
        dq 0x2725262423212220, 0x2f2d2e2c2b292a28
        dq 0x3735363433313230, 0x3f3d3e3c3b393a38

;;; ========================================================
;;; CODE
section .text

;;; arg 1 : pointer to DES OOO structure
;;; arg 2 : size in bytes
align 64
MKGLOBAL(des_x16_cbc_enc_avx512,function,internal)
des_x16_cbc_enc_avx512:
        GENERIC_DES_ENC DES
        ret

;;; arg 1 : pointer to DES OOO structure
;;; arg 2 : size in bytes
align 64
MKGLOBAL(des_x16_cbc_dec_avx512,function,internal)
des_x16_cbc_dec_avx512:
        GENERIC_DES_DEC DES
	ret

;;; arg 1 : pointer to DES OOO structure
;;; arg 2 : size in bytes
align 64
MKGLOBAL(des3_x16_cbc_enc_avx512,function,internal)
des3_x16_cbc_enc_avx512:
        GENERIC_DES_ENC 3DES
        ret

;;; arg 1 : pointer to DES OOO structure
;;; arg 2 : size in bytes
align 64
MKGLOBAL(des3_x16_cbc_dec_avx512,function,internal)
des3_x16_cbc_dec_avx512:
        GENERIC_DES_DEC 3DES
	ret

;;; arg 1 : pointer to DES OOO structure
;;; arg 2 : size in bytes
align 64
MKGLOBAL(docsis_des_x16_enc_avx512,function,internal)
docsis_des_x16_enc_avx512:
        GENERIC_DES_ENC DOCSIS
	ret

;;; arg 1 : pointer to DES OOO structure
;;; arg 2 : size in bytes
align 64
MKGLOBAL(docsis_des_x16_dec_avx512,function,internal)
docsis_des_x16_dec_avx512:
        GENERIC_DES_DEC DOCSIS
	ret

%ifdef LINUX
section .note.GNU-stack noalloc noexec nowrite progbits
%endif