diff options
Diffstat (limited to 'src/crypto/sha1/sha1block_amd64.s')
| -rw-r--r-- | src/crypto/sha1/sha1block_amd64.s | 1500 |
1 files changed, 1500 insertions, 0 deletions
diff --git a/src/crypto/sha1/sha1block_amd64.s b/src/crypto/sha1/sha1block_amd64.s new file mode 100644 index 0000000..42f03fb --- /dev/null +++ b/src/crypto/sha1/sha1block_amd64.s @@ -0,0 +1,1500 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// AVX2 version by Intel, same algorithm as code in Linux kernel: +// https://github.com/torvalds/linux/blob/master/arch/x86/crypto/sha1_avx2_x86_64_asm.S +// Authors: +// Ilya Albrekht <ilya.albrekht@intel.com> +// Maxim Locktyukhin <maxim.locktyukhin@intel.com> +// Ronen Zohar <ronen.zohar@intel.com> +// Chandramouli Narayanan <mouli@linux.intel.com> + + +#include "textflag.h" + +// SHA-1 block routine. See sha1block.go for Go equivalent. +// +// There are 80 rounds of 4 types: +// - rounds 0-15 are type 1 and load data (ROUND1 macro). +// - rounds 16-19 are type 1 and do not load data (ROUND1x macro). +// - rounds 20-39 are type 2 and do not load data (ROUND2 macro). +// - rounds 40-59 are type 3 and do not load data (ROUND3 macro). +// - rounds 60-79 are type 4 and do not load data (ROUND4 macro). +// +// Each round loads or shuffles the data, then computes a per-round +// function of b, c, d, and then mixes the result into and rotates the +// five registers a, b, c, d, e holding the intermediate results. +// +// The register rotation is implemented by rotating the arguments to +// the round macros instead of by explicit move instructions. + +#define LOAD(index) \ + MOVL (index*4)(SI), R10; \ + BSWAPL R10; \ + MOVL R10, (index*4)(SP) + +#define SHUFFLE(index) \ + MOVL (((index)&0xf)*4)(SP), R10; \ + XORL (((index-3)&0xf)*4)(SP), R10; \ + XORL (((index-8)&0xf)*4)(SP), R10; \ + XORL (((index-14)&0xf)*4)(SP), R10; \ + ROLL $1, R10; \ + MOVL R10, (((index)&0xf)*4)(SP) + +#define FUNC1(a, b, c, d, e) \ + MOVL d, R9; \ + XORL c, R9; \ + ANDL b, R9; \ + XORL d, R9 + +#define FUNC2(a, b, c, d, e) \ + MOVL b, R9; \ + XORL c, R9; \ + XORL d, R9 + +#define FUNC3(a, b, c, d, e) \ + MOVL b, R8; \ + ORL c, R8; \ + ANDL d, R8; \ + MOVL b, R9; \ + ANDL c, R9; \ + ORL R8, R9 + +#define FUNC4 FUNC2 + +#define MIX(a, b, c, d, e, const) \ + ROLL $30, b; \ + ADDL R9, e; \ + MOVL a, R8; \ + ROLL $5, R8; \ + LEAL const(e)(R10*1), e; \ + ADDL R8, e + +#define ROUND1(a, b, c, d, e, index) \ + LOAD(index); \ + FUNC1(a, b, c, d, e); \ + MIX(a, b, c, d, e, 0x5A827999) + +#define ROUND1x(a, b, c, d, e, index) \ + SHUFFLE(index); \ + FUNC1(a, b, c, d, e); \ + MIX(a, b, c, d, e, 0x5A827999) + +#define ROUND2(a, b, c, d, e, index) \ + SHUFFLE(index); \ + FUNC2(a, b, c, d, e); \ + MIX(a, b, c, d, e, 0x6ED9EBA1) + +#define ROUND3(a, b, c, d, e, index) \ + SHUFFLE(index); \ + FUNC3(a, b, c, d, e); \ + MIX(a, b, c, d, e, 0x8F1BBCDC) + +#define ROUND4(a, b, c, d, e, index) \ + SHUFFLE(index); \ + FUNC4(a, b, c, d, e); \ + MIX(a, b, c, d, e, 0xCA62C1D6) + +TEXT ·blockAMD64(SB),NOSPLIT,$64-32 + MOVQ dig+0(FP), BP + MOVQ p_base+8(FP), SI + MOVQ p_len+16(FP), DX + SHRQ $6, DX + SHLQ $6, DX + + LEAQ (SI)(DX*1), DI + MOVL (0*4)(BP), AX + MOVL (1*4)(BP), BX + MOVL (2*4)(BP), CX + MOVL (3*4)(BP), DX + MOVL (4*4)(BP), BP + + CMPQ SI, DI + JEQ end + +loop: + MOVL AX, R11 + MOVL BX, R12 + MOVL CX, R13 + MOVL DX, R14 + MOVL BP, R15 + + ROUND1(AX, BX, CX, DX, BP, 0) + ROUND1(BP, AX, BX, CX, DX, 1) + ROUND1(DX, BP, AX, BX, CX, 2) + ROUND1(CX, DX, BP, AX, BX, 3) + ROUND1(BX, CX, DX, BP, AX, 4) + ROUND1(AX, BX, CX, DX, BP, 5) + ROUND1(BP, AX, BX, CX, DX, 6) + ROUND1(DX, BP, AX, BX, CX, 7) + ROUND1(CX, DX, BP, AX, BX, 8) + ROUND1(BX, CX, DX, BP, AX, 9) + ROUND1(AX, BX, CX, DX, BP, 10) + ROUND1(BP, AX, BX, CX, DX, 11) + ROUND1(DX, BP, AX, BX, CX, 12) + ROUND1(CX, DX, BP, AX, BX, 13) + ROUND1(BX, CX, DX, BP, AX, 14) + ROUND1(AX, BX, CX, DX, BP, 15) + + ROUND1x(BP, AX, BX, CX, DX, 16) + ROUND1x(DX, BP, AX, BX, CX, 17) + ROUND1x(CX, DX, BP, AX, BX, 18) + ROUND1x(BX, CX, DX, BP, AX, 19) + + ROUND2(AX, BX, CX, DX, BP, 20) + ROUND2(BP, AX, BX, CX, DX, 21) + ROUND2(DX, BP, AX, BX, CX, 22) + ROUND2(CX, DX, BP, AX, BX, 23) + ROUND2(BX, CX, DX, BP, AX, 24) + ROUND2(AX, BX, CX, DX, BP, 25) + ROUND2(BP, AX, BX, CX, DX, 26) + ROUND2(DX, BP, AX, BX, CX, 27) + ROUND2(CX, DX, BP, AX, BX, 28) + ROUND2(BX, CX, DX, BP, AX, 29) + ROUND2(AX, BX, CX, DX, BP, 30) + ROUND2(BP, AX, BX, CX, DX, 31) + ROUND2(DX, BP, AX, BX, CX, 32) + ROUND2(CX, DX, BP, AX, BX, 33) + ROUND2(BX, CX, DX, BP, AX, 34) + ROUND2(AX, BX, CX, DX, BP, 35) + ROUND2(BP, AX, BX, CX, DX, 36) + ROUND2(DX, BP, AX, BX, CX, 37) + ROUND2(CX, DX, BP, AX, BX, 38) + ROUND2(BX, CX, DX, BP, AX, 39) + + ROUND3(AX, BX, CX, DX, BP, 40) + ROUND3(BP, AX, BX, CX, DX, 41) + ROUND3(DX, BP, AX, BX, CX, 42) + ROUND3(CX, DX, BP, AX, BX, 43) + ROUND3(BX, CX, DX, BP, AX, 44) + ROUND3(AX, BX, CX, DX, BP, 45) + ROUND3(BP, AX, BX, CX, DX, 46) + ROUND3(DX, BP, AX, BX, CX, 47) + ROUND3(CX, DX, BP, AX, BX, 48) + ROUND3(BX, CX, DX, BP, AX, 49) + ROUND3(AX, BX, CX, DX, BP, 50) + ROUND3(BP, AX, BX, CX, DX, 51) + ROUND3(DX, BP, AX, BX, CX, 52) + ROUND3(CX, DX, BP, AX, BX, 53) + ROUND3(BX, CX, DX, BP, AX, 54) + ROUND3(AX, BX, CX, DX, BP, 55) + ROUND3(BP, AX, BX, CX, DX, 56) + ROUND3(DX, BP, AX, BX, CX, 57) + ROUND3(CX, DX, BP, AX, BX, 58) + ROUND3(BX, CX, DX, BP, AX, 59) + + ROUND4(AX, BX, CX, DX, BP, 60) + ROUND4(BP, AX, BX, CX, DX, 61) + ROUND4(DX, BP, AX, BX, CX, 62) + ROUND4(CX, DX, BP, AX, BX, 63) + ROUND4(BX, CX, DX, BP, AX, 64) + ROUND4(AX, BX, CX, DX, BP, 65) + ROUND4(BP, AX, BX, CX, DX, 66) + ROUND4(DX, BP, AX, BX, CX, 67) + ROUND4(CX, DX, BP, AX, BX, 68) + ROUND4(BX, CX, DX, BP, AX, 69) + ROUND4(AX, BX, CX, DX, BP, 70) + ROUND4(BP, AX, BX, CX, DX, 71) + ROUND4(DX, BP, AX, BX, CX, 72) + ROUND4(CX, DX, BP, AX, BX, 73) + ROUND4(BX, CX, DX, BP, AX, 74) + ROUND4(AX, BX, CX, DX, BP, 75) + ROUND4(BP, AX, BX, CX, DX, 76) + ROUND4(DX, BP, AX, BX, CX, 77) + ROUND4(CX, DX, BP, AX, BX, 78) + ROUND4(BX, CX, DX, BP, AX, 79) + + ADDL R11, AX + ADDL R12, BX + ADDL R13, CX + ADDL R14, DX + ADDL R15, BP + + ADDQ $64, SI + CMPQ SI, DI + JB loop + +end: + MOVQ dig+0(FP), DI + MOVL AX, (0*4)(DI) + MOVL BX, (1*4)(DI) + MOVL CX, (2*4)(DI) + MOVL DX, (3*4)(DI) + MOVL BP, (4*4)(DI) + RET + + +// This is the implementation using AVX2, BMI1 and BMI2. It is based on: +// "SHA-1 implementation with Intel(R) AVX2 instruction set extensions" +// From http://software.intel.com/en-us/articles +// (look for improving-the-performance-of-the-secure-hash-algorithm-1) +// This implementation is 2x unrolled, and interleaves vector instructions, +// used to precompute W, with scalar computation of current round +// for optimal scheduling. + +// Trivial helper macros. +#define UPDATE_HASH(A,TB,C,D,E) \ + ADDL (R9), A \ + MOVL A, (R9) \ + ADDL 4(R9), TB \ + MOVL TB, 4(R9) \ + ADDL 8(R9), C \ + MOVL C, 8(R9) \ + ADDL 12(R9), D \ + MOVL D, 12(R9) \ + ADDL 16(R9), E \ + MOVL E, 16(R9) + + + +// Helper macros for PRECALC, which does precomputations +#define PRECALC_0(OFFSET) \ + VMOVDQU OFFSET(R10),X0 + +#define PRECALC_1(OFFSET) \ + VINSERTI128 $1, OFFSET(R13), Y0, Y0 + +#define PRECALC_2(YREG) \ + VPSHUFB Y10, Y0, YREG + +#define PRECALC_4(YREG,K_OFFSET) \ + VPADDD K_OFFSET(R8), YREG, Y0 + +#define PRECALC_7(OFFSET) \ + VMOVDQU Y0, (OFFSET*2)(R14) + + +// Message scheduling pre-compute for rounds 0-15 +// R13 is a pointer to even 64-byte block +// R10 is a pointer to odd 64-byte block +// R14 is a pointer to temp buffer +// X0 is used as temp register +// YREG is clobbered as part of computation +// OFFSET chooses 16 byte chunk within a block +// R8 is a pointer to constants block +// K_OFFSET chooses K constants relevant to this round +// X10 holds swap mask +#define PRECALC_00_15(OFFSET,YREG) \ + PRECALC_0(OFFSET) \ + PRECALC_1(OFFSET) \ + PRECALC_2(YREG) \ + PRECALC_4(YREG,0x0) \ + PRECALC_7(OFFSET) + + +// Helper macros for PRECALC_16_31 +#define PRECALC_16(REG_SUB_16,REG_SUB_12,REG_SUB_4,REG) \ + VPALIGNR $8, REG_SUB_16, REG_SUB_12, REG \ // w[i-14] + VPSRLDQ $4, REG_SUB_4, Y0 // w[i-3] + +#define PRECALC_17(REG_SUB_16,REG_SUB_8,REG) \ + VPXOR REG_SUB_8, REG, REG \ + VPXOR REG_SUB_16, Y0, Y0 + +#define PRECALC_18(REG) \ + VPXOR Y0, REG, REG \ + VPSLLDQ $12, REG, Y9 + +#define PRECALC_19(REG) \ + VPSLLD $1, REG, Y0 \ + VPSRLD $31, REG, REG + +#define PRECALC_20(REG) \ + VPOR REG, Y0, Y0 \ + VPSLLD $2, Y9, REG + +#define PRECALC_21(REG) \ + VPSRLD $30, Y9, Y9 \ + VPXOR REG, Y0, Y0 + +#define PRECALC_23(REG,K_OFFSET,OFFSET) \ + VPXOR Y9, Y0, REG \ + VPADDD K_OFFSET(R8), REG, Y0 \ + VMOVDQU Y0, (OFFSET)(R14) + +// Message scheduling pre-compute for rounds 16-31 +// calculating last 32 w[i] values in 8 XMM registers +// pre-calculate K+w[i] values and store to mem +// for later load by ALU add instruction. +// "brute force" vectorization for rounds 16-31 only +// due to w[i]->w[i-3] dependency. +// clobbers 5 input ymm registers REG_SUB* +// uses X0 and X9 as temp registers +// As always, R8 is a pointer to constants block +// and R14 is a pointer to temp buffer +#define PRECALC_16_31(REG,REG_SUB_4,REG_SUB_8,REG_SUB_12,REG_SUB_16,K_OFFSET,OFFSET) \ + PRECALC_16(REG_SUB_16,REG_SUB_12,REG_SUB_4,REG) \ + PRECALC_17(REG_SUB_16,REG_SUB_8,REG) \ + PRECALC_18(REG) \ + PRECALC_19(REG) \ + PRECALC_20(REG) \ + PRECALC_21(REG) \ + PRECALC_23(REG,K_OFFSET,OFFSET) + + +// Helper macros for PRECALC_32_79 +#define PRECALC_32(REG_SUB_8,REG_SUB_4) \ + VPALIGNR $8, REG_SUB_8, REG_SUB_4, Y0 + +#define PRECALC_33(REG_SUB_28,REG) \ + VPXOR REG_SUB_28, REG, REG + +#define PRECALC_34(REG_SUB_16) \ + VPXOR REG_SUB_16, Y0, Y0 + +#define PRECALC_35(REG) \ + VPXOR Y0, REG, REG + +#define PRECALC_36(REG) \ + VPSLLD $2, REG, Y0 + +#define PRECALC_37(REG) \ + VPSRLD $30, REG, REG \ + VPOR REG, Y0, REG + +#define PRECALC_39(REG,K_OFFSET,OFFSET) \ + VPADDD K_OFFSET(R8), REG, Y0 \ + VMOVDQU Y0, (OFFSET)(R14) + +// Message scheduling pre-compute for rounds 32-79 +// In SHA-1 specification we have: +// w[i] = (w[i-3] ^ w[i-8] ^ w[i-14] ^ w[i-16]) rol 1 +// Which is the same as: +// w[i] = (w[i-6] ^ w[i-16] ^ w[i-28] ^ w[i-32]) rol 2 +// This allows for more efficient vectorization, +// since w[i]->w[i-3] dependency is broken +#define PRECALC_32_79(REG,REG_SUB_4,REG_SUB_8,REG_SUB_16,REG_SUB_28,K_OFFSET,OFFSET) \ + PRECALC_32(REG_SUB_8,REG_SUB_4) \ + PRECALC_33(REG_SUB_28,REG) \ + PRECALC_34(REG_SUB_16) \ + PRECALC_35(REG) \ + PRECALC_36(REG) \ + PRECALC_37(REG) \ + PRECALC_39(REG,K_OFFSET,OFFSET) + +#define PRECALC \ + PRECALC_00_15(0,Y15) \ + PRECALC_00_15(0x10,Y14) \ + PRECALC_00_15(0x20,Y13) \ + PRECALC_00_15(0x30,Y12) \ + PRECALC_16_31(Y8,Y12,Y13,Y14,Y15,0,0x80) \ + PRECALC_16_31(Y7,Y8,Y12,Y13,Y14,0x20,0xa0) \ + PRECALC_16_31(Y5,Y7,Y8,Y12,Y13,0x20,0xc0) \ + PRECALC_16_31(Y3,Y5,Y7,Y8,Y12,0x20,0xe0) \ + PRECALC_32_79(Y15,Y3,Y5,Y8,Y14,0x20,0x100) \ + PRECALC_32_79(Y14,Y15,Y3,Y7,Y13,0x20,0x120) \ + PRECALC_32_79(Y13,Y14,Y15,Y5,Y12,0x40,0x140) \ + PRECALC_32_79(Y12,Y13,Y14,Y3,Y8,0x40,0x160) \ + PRECALC_32_79(Y8,Y12,Y13,Y15,Y7,0x40,0x180) \ + PRECALC_32_79(Y7,Y8,Y12,Y14,Y5,0x40,0x1a0) \ + PRECALC_32_79(Y5,Y7,Y8,Y13,Y3,0x40,0x1c0) \ + PRECALC_32_79(Y3,Y5,Y7,Y12,Y15,0x60,0x1e0) \ + PRECALC_32_79(Y15,Y3,Y5,Y8,Y14,0x60,0x200) \ + PRECALC_32_79(Y14,Y15,Y3,Y7,Y13,0x60,0x220) \ + PRECALC_32_79(Y13,Y14,Y15,Y5,Y12,0x60,0x240) \ + PRECALC_32_79(Y12,Y13,Y14,Y3,Y8,0x60,0x260) + +// Macros calculating individual rounds have general form +// CALC_ROUND_PRE + PRECALC_ROUND + CALC_ROUND_POST +// CALC_ROUND_{PRE,POST} macros follow + +#define CALC_F1_PRE(OFFSET,REG_A,REG_B,REG_C,REG_E) \ + ADDL OFFSET(R15),REG_E \ + ANDNL REG_C,REG_A,BP \ + LEAL (REG_E)(REG_B*1), REG_E \ // Add F from the previous round + RORXL $0x1b, REG_A, R12 \ + RORXL $2, REG_A, REG_B // for next round + +// Calculate F for the next round +#define CALC_F1_POST(REG_A,REG_B,REG_E) \ + ANDL REG_B,REG_A \ // b&c + XORL BP, REG_A \ // F1 = (b&c) ^ (~b&d) + LEAL (REG_E)(R12*1), REG_E // E += A >>> 5 + + +// Registers are cyclically rotated DX -> AX -> DI -> SI -> BX -> CX +#define CALC_0 \ + MOVL SI, BX \ // Precalculating first round + RORXL $2, SI, SI \ + ANDNL AX, BX, BP \ + ANDL DI, BX \ + XORL BP, BX \ + CALC_F1_PRE(0x0,CX,BX,DI,DX) \ + PRECALC_0(0x80) \ + CALC_F1_POST(CX,SI,DX) + +#define CALC_1 \ + CALC_F1_PRE(0x4,DX,CX,SI,AX) \ + PRECALC_1(0x80) \ + CALC_F1_POST(DX,BX,AX) + +#define CALC_2 \ + CALC_F1_PRE(0x8,AX,DX,BX,DI) \ + PRECALC_2(Y15) \ + CALC_F1_POST(AX,CX,DI) + +#define CALC_3 \ + CALC_F1_PRE(0xc,DI,AX,CX,SI) \ + CALC_F1_POST(DI,DX,SI) + +#define CALC_4 \ + CALC_F1_PRE(0x20,SI,DI,DX,BX) \ + PRECALC_4(Y15,0x0) \ + CALC_F1_POST(SI,AX,BX) + +#define CALC_5 \ + CALC_F1_PRE(0x24,BX,SI,AX,CX) \ + CALC_F1_POST(BX,DI,CX) + +#define CALC_6 \ + CALC_F1_PRE(0x28,CX,BX,DI,DX) \ + CALC_F1_POST(CX,SI,DX) + +#define CALC_7 \ + CALC_F1_PRE(0x2c,DX,CX,SI,AX) \ + PRECALC_7(0x0) \ + CALC_F1_POST(DX,BX,AX) + +#define CALC_8 \ + CALC_F1_PRE(0x40,AX,DX,BX,DI) \ + PRECALC_0(0x90) \ + CALC_F1_POST(AX,CX,DI) + +#define CALC_9 \ + CALC_F1_PRE(0x44,DI,AX,CX,SI) \ + PRECALC_1(0x90) \ + CALC_F1_POST(DI,DX,SI) + +#define CALC_10 \ + CALC_F1_PRE(0x48,SI,DI,DX,BX) \ + PRECALC_2(Y14) \ + CALC_F1_POST(SI,AX,BX) + +#define CALC_11 \ + CALC_F1_PRE(0x4c,BX,SI,AX,CX) \ + CALC_F1_POST(BX,DI,CX) + +#define CALC_12 \ + CALC_F1_PRE(0x60,CX,BX,DI,DX) \ + PRECALC_4(Y14,0x0) \ + CALC_F1_POST(CX,SI,DX) + +#define CALC_13 \ + CALC_F1_PRE(0x64,DX,CX,SI,AX) \ + CALC_F1_POST(DX,BX,AX) + +#define CALC_14 \ + CALC_F1_PRE(0x68,AX,DX,BX,DI) \ + CALC_F1_POST(AX,CX,DI) + +#define CALC_15 \ + CALC_F1_PRE(0x6c,DI,AX,CX,SI) \ + PRECALC_7(0x10) \ + CALC_F1_POST(DI,DX,SI) + +#define CALC_16 \ + CALC_F1_PRE(0x80,SI,DI,DX,BX) \ + PRECALC_0(0xa0) \ + CALC_F1_POST(SI,AX,BX) + +#define CALC_17 \ + CALC_F1_PRE(0x84,BX,SI,AX,CX) \ + PRECALC_1(0xa0) \ + CALC_F1_POST(BX,DI,CX) + +#define CALC_18 \ + CALC_F1_PRE(0x88,CX,BX,DI,DX) \ + PRECALC_2(Y13) \ + CALC_F1_POST(CX,SI,DX) + + +#define CALC_F2_PRE(OFFSET,REG_A,REG_B,REG_E) \ + ADDL OFFSET(R15),REG_E \ + LEAL (REG_E)(REG_B*1), REG_E \ // Add F from the previous round + RORXL $0x1b, REG_A, R12 \ + RORXL $2, REG_A, REG_B // for next round + +#define CALC_F2_POST(REG_A,REG_B,REG_C,REG_E) \ + XORL REG_B, REG_A \ + ADDL R12, REG_E \ + XORL REG_C, REG_A + +#define CALC_19 \ + CALC_F2_PRE(0x8c,DX,CX,AX) \ + CALC_F2_POST(DX,BX,SI,AX) + +#define CALC_20 \ + CALC_F2_PRE(0xa0,AX,DX,DI) \ + PRECALC_4(Y13,0x0) \ + CALC_F2_POST(AX,CX,BX,DI) + +#define CALC_21 \ + CALC_F2_PRE(0xa4,DI,AX,SI) \ + CALC_F2_POST(DI,DX,CX,SI) + +#define CALC_22 \ + CALC_F2_PRE(0xa8,SI,DI,BX) \ + CALC_F2_POST(SI,AX,DX,BX) + +#define CALC_23 \ + CALC_F2_PRE(0xac,BX,SI,CX) \ + PRECALC_7(0x20) \ + CALC_F2_POST(BX,DI,AX,CX) + +#define CALC_24 \ + CALC_F2_PRE(0xc0,CX,BX,DX) \ + PRECALC_0(0xb0) \ + CALC_F2_POST(CX,SI,DI,DX) + +#define CALC_25 \ + CALC_F2_PRE(0xc4,DX,CX,AX) \ + PRECALC_1(0xb0) \ + CALC_F2_POST(DX,BX,SI,AX) + +#define CALC_26 \ + CALC_F2_PRE(0xc8,AX,DX,DI) \ + PRECALC_2(Y12) \ + CALC_F2_POST(AX,CX,BX,DI) + +#define CALC_27 \ + CALC_F2_PRE(0xcc,DI,AX,SI) \ + CALC_F2_POST(DI,DX,CX,SI) + +#define CALC_28 \ + CALC_F2_PRE(0xe0,SI,DI,BX) \ + PRECALC_4(Y12,0x0) \ + CALC_F2_POST(SI,AX,DX,BX) + +#define CALC_29 \ + CALC_F2_PRE(0xe4,BX,SI,CX) \ + CALC_F2_POST(BX,DI,AX,CX) + +#define CALC_30 \ + CALC_F2_PRE(0xe8,CX,BX,DX) \ + CALC_F2_POST(CX,SI,DI,DX) + +#define CALC_31 \ + CALC_F2_PRE(0xec,DX,CX,AX) \ + PRECALC_7(0x30) \ + CALC_F2_POST(DX,BX,SI,AX) + +#define CALC_32 \ + CALC_F2_PRE(0x100,AX,DX,DI) \ + PRECALC_16(Y15,Y14,Y12,Y8) \ + CALC_F2_POST(AX,CX,BX,DI) + +#define CALC_33 \ + CALC_F2_PRE(0x104,DI,AX,SI) \ + PRECALC_17(Y15,Y13,Y8) \ + CALC_F2_POST(DI,DX,CX,SI) + +#define CALC_34 \ + CALC_F2_PRE(0x108,SI,DI,BX) \ + PRECALC_18(Y8) \ + CALC_F2_POST(SI,AX,DX,BX) + +#define CALC_35 \ + CALC_F2_PRE(0x10c,BX,SI,CX) \ + PRECALC_19(Y8) \ + CALC_F2_POST(BX,DI,AX,CX) + +#define CALC_36 \ + CALC_F2_PRE(0x120,CX,BX,DX) \ + PRECALC_20(Y8) \ + CALC_F2_POST(CX,SI,DI,DX) + +#define CALC_37 \ + CALC_F2_PRE(0x124,DX,CX,AX) \ + PRECALC_21(Y8) \ + CALC_F2_POST(DX,BX,SI,AX) + +#define CALC_38 \ + CALC_F2_PRE(0x128,AX,DX,DI) \ + CALC_F2_POST(AX,CX,BX,DI) + + +#define CALC_F3_PRE(OFFSET,REG_E) \ + ADDL OFFSET(R15),REG_E + +#define CALC_F3_POST(REG_A,REG_B,REG_C,REG_E,REG_TB) \ + LEAL (REG_E)(REG_TB*1), REG_E \ // Add F from the previous round + MOVL REG_B, BP \ + ORL REG_A, BP \ + RORXL $0x1b, REG_A, R12 \ + RORXL $2, REG_A, REG_TB \ + ANDL REG_C, BP \ // Calculate F for the next round + ANDL REG_B, REG_A \ + ORL BP, REG_A \ + ADDL R12, REG_E + +#define CALC_39 \ + CALC_F3_PRE(0x12c,SI) \ + PRECALC_23(Y8,0x0,0x80) \ + CALC_F3_POST(DI,DX,CX,SI,AX) + +#define CALC_40 \ + CALC_F3_PRE(0x140,BX) \ + PRECALC_16(Y14,Y13,Y8,Y7) \ + CALC_F3_POST(SI,AX,DX,BX,DI) + +#define CALC_41 \ + CALC_F3_PRE(0x144,CX) \ + PRECALC_17(Y14,Y12,Y7) \ + CALC_F3_POST(BX,DI,AX,CX,SI) + +#define CALC_42 \ + CALC_F3_PRE(0x148,DX) \ + PRECALC_18(Y7) \ + CALC_F3_POST(CX,SI,DI,DX,BX) + +#define CALC_43 \ + CALC_F3_PRE(0x14c,AX) \ + PRECALC_19(Y7) \ + CALC_F3_POST(DX,BX,SI,AX,CX) + +#define CALC_44 \ + CALC_F3_PRE(0x160,DI) \ + PRECALC_20(Y7) \ + CALC_F3_POST(AX,CX,BX,DI,DX) + +#define CALC_45 \ + CALC_F3_PRE(0x164,SI) \ + PRECALC_21(Y7) \ + CALC_F3_POST(DI,DX,CX,SI,AX) + +#define CALC_46 \ + CALC_F3_PRE(0x168,BX) \ + CALC_F3_POST(SI,AX,DX,BX,DI) + +#define CALC_47 \ + CALC_F3_PRE(0x16c,CX) \ + VPXOR Y9, Y0, Y7 \ + VPADDD 0x20(R8), Y7, Y0 \ + VMOVDQU Y0, 0xa0(R14) \ + CALC_F3_POST(BX,DI,AX,CX,SI) + +#define CALC_48 \ + CALC_F3_PRE(0x180,DX) \ + PRECALC_16(Y13,Y12,Y7,Y5) \ + CALC_F3_POST(CX,SI,DI,DX,BX) + +#define CALC_49 \ + CALC_F3_PRE(0x184,AX) \ + PRECALC_17(Y13,Y8,Y5) \ + CALC_F3_POST(DX,BX,SI,AX,CX) + +#define CALC_50 \ + CALC_F3_PRE(0x188,DI) \ + PRECALC_18(Y5) \ + CALC_F3_POST(AX,CX,BX,DI,DX) + +#define CALC_51 \ + CALC_F3_PRE(0x18c,SI) \ + PRECALC_19(Y5) \ + CALC_F3_POST(DI,DX,CX,SI,AX) + +#define CALC_52 \ + CALC_F3_PRE(0x1a0,BX) \ + PRECALC_20(Y5) \ + CALC_F3_POST(SI,AX,DX,BX,DI) + +#define CALC_53 \ + CALC_F3_PRE(0x1a4,CX) \ + PRECALC_21(Y5) \ + CALC_F3_POST(BX,DI,AX,CX,SI) + +#define CALC_54 \ + CALC_F3_PRE(0x1a8,DX) \ + CALC_F3_POST(CX,SI,DI,DX,BX) + +#define CALC_55 \ + CALC_F3_PRE(0x1ac,AX) \ + PRECALC_23(Y5,0x20,0xc0) \ + CALC_F3_POST(DX,BX,SI,AX,CX) + +#define CALC_56 \ + CALC_F3_PRE(0x1c0,DI) \ + PRECALC_16(Y12,Y8,Y5,Y3) \ + CALC_F3_POST(AX,CX,BX,DI,DX) + +#define CALC_57 \ + CALC_F3_PRE(0x1c4,SI) \ + PRECALC_17(Y12,Y7,Y3) \ + CALC_F3_POST(DI,DX,CX,SI,AX) + +#define CALC_58 \ + CALC_F3_PRE(0x1c8,BX) \ + PRECALC_18(Y3) \ + CALC_F3_POST(SI,AX,DX,BX,DI) + +#define CALC_59 \ + CALC_F2_PRE(0x1cc,BX,SI,CX) \ + PRECALC_19(Y3) \ + CALC_F2_POST(BX,DI,AX,CX) + +#define CALC_60 \ + CALC_F2_PRE(0x1e0,CX,BX,DX) \ + PRECALC_20(Y3) \ + CALC_F2_POST(CX,SI,DI,DX) + +#define CALC_61 \ + CALC_F2_PRE(0x1e4,DX,CX,AX) \ + PRECALC_21(Y3) \ + CALC_F2_POST(DX,BX,SI,AX) + +#define CALC_62 \ + CALC_F2_PRE(0x1e8,AX,DX,DI) \ + CALC_F2_POST(AX,CX,BX,DI) + +#define CALC_63 \ + CALC_F2_PRE(0x1ec,DI,AX,SI) \ + PRECALC_23(Y3,0x20,0xe0) \ + CALC_F2_POST(DI,DX,CX,SI) + +#define CALC_64 \ + CALC_F2_PRE(0x200,SI,DI,BX) \ + PRECALC_32(Y5,Y3) \ + CALC_F2_POST(SI,AX,DX,BX) + +#define CALC_65 \ + CALC_F2_PRE(0x204,BX,SI,CX) \ + PRECALC_33(Y14,Y15) \ + CALC_F2_POST(BX,DI,AX,CX) + +#define CALC_66 \ + CALC_F2_PRE(0x208,CX,BX,DX) \ + PRECALC_34(Y8) \ + CALC_F2_POST(CX,SI,DI,DX) + +#define CALC_67 \ + CALC_F2_PRE(0x20c,DX,CX,AX) \ + PRECALC_35(Y15) \ + CALC_F2_POST(DX,BX,SI,AX) + +#define CALC_68 \ + CALC_F2_PRE(0x220,AX,DX,DI) \ + PRECALC_36(Y15) \ + CALC_F2_POST(AX,CX,BX,DI) + +#define CALC_69 \ + CALC_F2_PRE(0x224,DI,AX,SI) \ + PRECALC_37(Y15) \ + CALC_F2_POST(DI,DX,CX,SI) + +#define CALC_70 \ + CALC_F2_PRE(0x228,SI,DI,BX) \ + CALC_F2_POST(SI,AX,DX,BX) + +#define CALC_71 \ + CALC_F2_PRE(0x22c,BX,SI,CX) \ + PRECALC_39(Y15,0x20,0x100) \ + CALC_F2_POST(BX,DI,AX,CX) + +#define CALC_72 \ + CALC_F2_PRE(0x240,CX,BX,DX) \ + PRECALC_32(Y3,Y15) \ + CALC_F2_POST(CX,SI,DI,DX) + +#define CALC_73 \ + CALC_F2_PRE(0x244,DX,CX,AX) \ + PRECALC_33(Y13,Y14) \ + CALC_F2_POST(DX,BX,SI,AX) + +#define CALC_74 \ + CALC_F2_PRE(0x248,AX,DX,DI) \ + PRECALC_34(Y7) \ + CALC_F2_POST(AX,CX,BX,DI) + +#define CALC_75 \ + CALC_F2_PRE(0x24c,DI,AX,SI) \ + PRECALC_35(Y14) \ + CALC_F2_POST(DI,DX,CX,SI) + +#define CALC_76 \ + CALC_F2_PRE(0x260,SI,DI,BX) \ + PRECALC_36(Y14) \ + CALC_F2_POST(SI,AX,DX,BX) + +#define CALC_77 \ + CALC_F2_PRE(0x264,BX,SI,CX) \ + PRECALC_37(Y14) \ + CALC_F2_POST(BX,DI,AX,CX) + +#define CALC_78 \ + CALC_F2_PRE(0x268,CX,BX,DX) \ + CALC_F2_POST(CX,SI,DI,DX) + +#define CALC_79 \ + ADDL 0x26c(R15), AX \ + LEAL (AX)(CX*1), AX \ + RORXL $0x1b, DX, R12 \ + PRECALC_39(Y14,0x20,0x120) \ + ADDL R12, AX + +// Similar to CALC_0 +#define CALC_80 \ + MOVL CX, DX \ + RORXL $2, CX, CX \ + ANDNL SI, DX, BP \ + ANDL BX, DX \ + XORL BP, DX \ + CALC_F1_PRE(0x10,AX,DX,BX,DI) \ + PRECALC_32(Y15,Y14) \ + CALC_F1_POST(AX,CX,DI) + +#define CALC_81 \ + CALC_F1_PRE(0x14,DI,AX,CX,SI) \ + PRECALC_33(Y12,Y13) \ + CALC_F1_POST(DI,DX,SI) + +#define CALC_82 \ + CALC_F1_PRE(0x18,SI,DI,DX,BX) \ + PRECALC_34(Y5) \ + CALC_F1_POST(SI,AX,BX) + +#define CALC_83 \ + CALC_F1_PRE(0x1c,BX,SI,AX,CX) \ + PRECALC_35(Y13) \ + CALC_F1_POST(BX,DI,CX) + +#define CALC_84 \ + CALC_F1_PRE(0x30,CX,BX,DI,DX) \ + PRECALC_36(Y13) \ + CALC_F1_POST(CX,SI,DX) + +#define CALC_85 \ + CALC_F1_PRE(0x34,DX,CX,SI,AX) \ + PRECALC_37(Y13) \ + CALC_F1_POST(DX,BX,AX) + +#define CALC_86 \ + CALC_F1_PRE(0x38,AX,DX,BX,DI) \ + CALC_F1_POST(AX,CX,DI) + +#define CALC_87 \ + CALC_F1_PRE(0x3c,DI,AX,CX,SI) \ + PRECALC_39(Y13,0x40,0x140) \ + CALC_F1_POST(DI,DX,SI) + +#define CALC_88 \ + CALC_F1_PRE(0x50,SI,DI,DX,BX) \ + PRECALC_32(Y14,Y13) \ + CALC_F1_POST(SI,AX,BX) + +#define CALC_89 \ + CALC_F1_PRE(0x54,BX,SI,AX,CX) \ + PRECALC_33(Y8,Y12) \ + CALC_F1_POST(BX,DI,CX) + +#define CALC_90 \ + CALC_F1_PRE(0x58,CX,BX,DI,DX) \ + PRECALC_34(Y3) \ + CALC_F1_POST(CX,SI,DX) + +#define CALC_91 \ + CALC_F1_PRE(0x5c,DX,CX,SI,AX) \ + PRECALC_35(Y12) \ + CALC_F1_POST(DX,BX,AX) + +#define CALC_92 \ + CALC_F1_PRE(0x70,AX,DX,BX,DI) \ + PRECALC_36(Y12) \ + CALC_F1_POST(AX,CX,DI) + +#define CALC_93 \ + CALC_F1_PRE(0x74,DI,AX,CX,SI) \ + PRECALC_37(Y12) \ + CALC_F1_POST(DI,DX,SI) + +#define CALC_94 \ + CALC_F1_PRE(0x78,SI,DI,DX,BX) \ + CALC_F1_POST(SI,AX,BX) + +#define CALC_95 \ + CALC_F1_PRE(0x7c,BX,SI,AX,CX) \ + PRECALC_39(Y12,0x40,0x160) \ + CALC_F1_POST(BX,DI,CX) + +#define CALC_96 \ + CALC_F1_PRE(0x90,CX,BX,DI,DX) \ + PRECALC_32(Y13,Y12) \ + CALC_F1_POST(CX,SI,DX) + +#define CALC_97 \ + CALC_F1_PRE(0x94,DX,CX,SI,AX) \ + PRECALC_33(Y7,Y8) \ + CALC_F1_POST(DX,BX,AX) + +#define CALC_98 \ + CALC_F1_PRE(0x98,AX,DX,BX,DI) \ + PRECALC_34(Y15) \ + CALC_F1_POST(AX,CX,DI) + +#define CALC_99 \ + CALC_F2_PRE(0x9c,DI,AX,SI) \ + PRECALC_35(Y8) \ + CALC_F2_POST(DI,DX,CX,SI) + +#define CALC_100 \ + CALC_F2_PRE(0xb0,SI,DI,BX) \ + PRECALC_36(Y8) \ + CALC_F2_POST(SI,AX,DX,BX) + +#define CALC_101 \ + CALC_F2_PRE(0xb4,BX,SI,CX) \ + PRECALC_37(Y8) \ + CALC_F2_POST(BX,DI,AX,CX) + +#define CALC_102 \ + CALC_F2_PRE(0xb8,CX,BX,DX) \ + CALC_F2_POST(CX,SI,DI,DX) + +#define CALC_103 \ + CALC_F2_PRE(0xbc,DX,CX,AX) \ + PRECALC_39(Y8,0x40,0x180) \ + CALC_F2_POST(DX,BX,SI,AX) + +#define CALC_104 \ + CALC_F2_PRE(0xd0,AX,DX,DI) \ + PRECALC_32(Y12,Y8) \ + CALC_F2_POST(AX,CX,BX,DI) + +#define CALC_105 \ + CALC_F2_PRE(0xd4,DI,AX,SI) \ + PRECALC_33(Y5,Y7) \ + CALC_F2_POST(DI,DX,CX,SI) + +#define CALC_106 \ + CALC_F2_PRE(0xd8,SI,DI,BX) \ + PRECALC_34(Y14) \ + CALC_F2_POST(SI,AX,DX,BX) + +#define CALC_107 \ + CALC_F2_PRE(0xdc,BX,SI,CX) \ + PRECALC_35(Y7) \ + CALC_F2_POST(BX,DI,AX,CX) + +#define CALC_108 \ + CALC_F2_PRE(0xf0,CX,BX,DX) \ + PRECALC_36(Y7) \ + CALC_F2_POST(CX,SI,DI,DX) + +#define CALC_109 \ + CALC_F2_PRE(0xf4,DX,CX,AX) \ + PRECALC_37(Y7) \ + CALC_F2_POST(DX,BX,SI,AX) + +#define CALC_110 \ + CALC_F2_PRE(0xf8,AX,DX,DI) \ + CALC_F2_POST(AX,CX,BX,DI) + +#define CALC_111 \ + CALC_F2_PRE(0xfc,DI,AX,SI) \ + PRECALC_39(Y7,0x40,0x1a0) \ + CALC_F2_POST(DI,DX,CX,SI) + +#define CALC_112 \ + CALC_F2_PRE(0x110,SI,DI,BX) \ + PRECALC_32(Y8,Y7) \ + CALC_F2_POST(SI,AX,DX,BX) + +#define CALC_113 \ + CALC_F2_PRE(0x114,BX,SI,CX) \ + PRECALC_33(Y3,Y5) \ + CALC_F2_POST(BX,DI,AX,CX) + +#define CALC_114 \ + CALC_F2_PRE(0x118,CX,BX,DX) \ + PRECALC_34(Y13) \ + CALC_F2_POST(CX,SI,DI,DX) + +#define CALC_115 \ + CALC_F2_PRE(0x11c,DX,CX,AX) \ + PRECALC_35(Y5) \ + CALC_F2_POST(DX,BX,SI,AX) + +#define CALC_116 \ + CALC_F2_PRE(0x130,AX,DX,DI) \ + PRECALC_36(Y5) \ + CALC_F2_POST(AX,CX,BX,DI) + +#define CALC_117 \ + CALC_F2_PRE(0x134,DI,AX,SI) \ + PRECALC_37(Y5) \ + CALC_F2_POST(DI,DX,CX,SI) + +#define CALC_118 \ + CALC_F2_PRE(0x138,SI,DI,BX) \ + CALC_F2_POST(SI,AX,DX,BX) + +#define CALC_119 \ + CALC_F3_PRE(0x13c,CX) \ + PRECALC_39(Y5,0x40,0x1c0) \ + CALC_F3_POST(BX,DI,AX,CX,SI) + +#define CALC_120 \ + CALC_F3_PRE(0x150,DX) \ + PRECALC_32(Y7,Y5) \ + CALC_F3_POST(CX,SI,DI,DX,BX) + +#define CALC_121 \ + CALC_F3_PRE(0x154,AX) \ + PRECALC_33(Y15,Y3) \ + CALC_F3_POST(DX,BX,SI,AX,CX) + +#define CALC_122 \ + CALC_F3_PRE(0x158,DI) \ + PRECALC_34(Y12) \ + CALC_F3_POST(AX,CX,BX,DI,DX) + +#define CALC_123 \ + CALC_F3_PRE(0x15c,SI) \ + PRECALC_35(Y3) \ + CALC_F3_POST(DI,DX,CX,SI,AX) + +#define CALC_124 \ + CALC_F3_PRE(0x170,BX) \ + PRECALC_36(Y3) \ + CALC_F3_POST(SI,AX,DX,BX,DI) + +#define CALC_125 \ + CALC_F3_PRE(0x174,CX) \ + PRECALC_37(Y3) \ + CALC_F3_POST(BX,DI,AX,CX,SI) + +#define CALC_126 \ + CALC_F3_PRE(0x178,DX) \ + CALC_F3_POST(CX,SI,DI,DX,BX) + +#define CALC_127 \ + CALC_F3_PRE(0x17c,AX) \ + PRECALC_39(Y3,0x60,0x1e0) \ + CALC_F3_POST(DX,BX,SI,AX,CX) + +#define CALC_128 \ + CALC_F3_PRE(0x190,DI) \ + PRECALC_32(Y5,Y3) \ + CALC_F3_POST(AX,CX,BX,DI,DX) + +#define CALC_129 \ + CALC_F3_PRE(0x194,SI) \ + PRECALC_33(Y14,Y15) \ + CALC_F3_POST(DI,DX,CX,SI,AX) + +#define CALC_130 \ + CALC_F3_PRE(0x198,BX) \ + PRECALC_34(Y8) \ + CALC_F3_POST(SI,AX,DX,BX,DI) + +#define CALC_131 \ + CALC_F3_PRE(0x19c,CX) \ + PRECALC_35(Y15) \ + CALC_F3_POST(BX,DI,AX,CX,SI) + +#define CALC_132 \ + CALC_F3_PRE(0x1b0,DX) \ + PRECALC_36(Y15) \ + CALC_F3_POST(CX,SI,DI,DX,BX) + +#define CALC_133 \ + CALC_F3_PRE(0x1b4,AX) \ + PRECALC_37(Y15) \ + CALC_F3_POST(DX,BX,SI,AX,CX) + +#define CALC_134 \ + CALC_F3_PRE(0x1b8,DI) \ + CALC_F3_POST(AX,CX,BX,DI,DX) + +#define CALC_135 \ + CALC_F3_PRE(0x1bc,SI) \ + PRECALC_39(Y15,0x60,0x200) \ + CALC_F3_POST(DI,DX,CX,SI,AX) + +#define CALC_136 \ + CALC_F3_PRE(0x1d0,BX) \ + PRECALC_32(Y3,Y15) \ + CALC_F3_POST(SI,AX,DX,BX,DI) + +#define CALC_137 \ + CALC_F3_PRE(0x1d4,CX) \ + PRECALC_33(Y13,Y14) \ + CALC_F3_POST(BX,DI,AX,CX,SI) + +#define CALC_138 \ + CALC_F3_PRE(0x1d8,DX) \ + PRECALC_34(Y7) \ + CALC_F3_POST(CX,SI,DI,DX,BX) + +#define CALC_139 \ + CALC_F2_PRE(0x1dc,DX,CX,AX) \ + PRECALC_35(Y14) \ + CALC_F2_POST(DX,BX,SI,AX) + +#define CALC_140 \ + CALC_F2_PRE(0x1f0,AX,DX,DI) \ + PRECALC_36(Y14) \ + CALC_F2_POST(AX,CX,BX,DI) + +#define CALC_141 \ + CALC_F2_PRE(0x1f4,DI,AX,SI) \ + PRECALC_37(Y14) \ + CALC_F2_POST(DI,DX,CX,SI) + +#define CALC_142 \ + CALC_F2_PRE(0x1f8,SI,DI,BX) \ + CALC_F2_POST(SI,AX,DX,BX) + +#define CALC_143 \ + CALC_F2_PRE(0x1fc,BX,SI,CX) \ + PRECALC_39(Y14,0x60,0x220) \ + CALC_F2_POST(BX,DI,AX,CX) + +#define CALC_144 \ + CALC_F2_PRE(0x210,CX,BX,DX) \ + PRECALC_32(Y15,Y14) \ + CALC_F2_POST(CX,SI,DI,DX) + +#define CALC_145 \ + CALC_F2_PRE(0x214,DX,CX,AX) \ + PRECALC_33(Y12,Y13) \ + CALC_F2_POST(DX,BX,SI,AX) + +#define CALC_146 \ + CALC_F2_PRE(0x218,AX,DX,DI) \ + PRECALC_34(Y5) \ + CALC_F2_POST(AX,CX,BX,DI) + +#define CALC_147 \ + CALC_F2_PRE(0x21c,DI,AX,SI) \ + PRECALC_35(Y13) \ + CALC_F2_POST(DI,DX,CX,SI) + +#define CALC_148 \ + CALC_F2_PRE(0x230,SI,DI,BX) \ + PRECALC_36(Y13) \ + CALC_F2_POST(SI,AX,DX,BX) + +#define CALC_149 \ + CALC_F2_PRE(0x234,BX,SI,CX) \ + PRECALC_37(Y13) \ + CALC_F2_POST(BX,DI,AX,CX) + +#define CALC_150 \ + CALC_F2_PRE(0x238,CX,BX,DX) \ + CALC_F2_POST(CX,SI,DI,DX) + +#define CALC_151 \ + CALC_F2_PRE(0x23c,DX,CX,AX) \ + PRECALC_39(Y13,0x60,0x240) \ + CALC_F2_POST(DX,BX,SI,AX) + +#define CALC_152 \ + CALC_F2_PRE(0x250,AX,DX,DI) \ + PRECALC_32(Y14,Y13) \ + CALC_F2_POST(AX,CX,BX,DI) + +#define CALC_153 \ + CALC_F2_PRE(0x254,DI,AX,SI) \ + PRECALC_33(Y8,Y12) \ + CALC_F2_POST(DI,DX,CX,SI) + +#define CALC_154 \ + CALC_F2_PRE(0x258,SI,DI,BX) \ + PRECALC_34(Y3) \ + CALC_F2_POST(SI,AX,DX,BX) + +#define CALC_155 \ + CALC_F2_PRE(0x25c,BX,SI,CX) \ + PRECALC_35(Y12) \ + CALC_F2_POST(BX,DI,AX,CX) + +#define CALC_156 \ + CALC_F2_PRE(0x270,CX,BX,DX) \ + PRECALC_36(Y12) \ + CALC_F2_POST(CX,SI,DI,DX) + +#define CALC_157 \ + CALC_F2_PRE(0x274,DX,CX,AX) \ + PRECALC_37(Y12) \ + CALC_F2_POST(DX,BX,SI,AX) + +#define CALC_158 \ + CALC_F2_PRE(0x278,AX,DX,DI) \ + CALC_F2_POST(AX,CX,BX,DI) + +#define CALC_159 \ + ADDL 0x27c(R15),SI \ + LEAL (SI)(AX*1), SI \ + RORXL $0x1b, DI, R12 \ + PRECALC_39(Y12,0x60,0x260) \ + ADDL R12, SI + + + +#define CALC \ + MOVL (R9), CX \ + MOVL 4(R9), SI \ + MOVL 8(R9), DI \ + MOVL 12(R9), AX \ + MOVL 16(R9), DX \ + MOVQ SP, R14 \ + LEAQ (2*4*80+32)(SP), R15 \ + PRECALC \ // Precalc WK for first 2 blocks + XCHGQ R15, R14 \ +loop: \ // this loops is unrolled + CMPQ R10, R8 \ // we use R8 value (set below) as a signal of a last block + JNE begin \ + VZEROUPPER \ + RET \ +begin: \ + CALC_0 \ + CALC_1 \ + CALC_2 \ + CALC_3 \ + CALC_4 \ + CALC_5 \ + CALC_6 \ + CALC_7 \ + CALC_8 \ + CALC_9 \ + CALC_10 \ + CALC_11 \ + CALC_12 \ + CALC_13 \ + CALC_14 \ + CALC_15 \ + CALC_16 \ + CALC_17 \ + CALC_18 \ + CALC_19 \ + CALC_20 \ + CALC_21 \ + CALC_22 \ + CALC_23 \ + CALC_24 \ + CALC_25 \ + CALC_26 \ + CALC_27 \ + CALC_28 \ + CALC_29 \ + CALC_30 \ + CALC_31 \ + CALC_32 \ + CALC_33 \ + CALC_34 \ + CALC_35 \ + CALC_36 \ + CALC_37 \ + CALC_38 \ + CALC_39 \ + CALC_40 \ + CALC_41 \ + CALC_42 \ + CALC_43 \ + CALC_44 \ + CALC_45 \ + CALC_46 \ + CALC_47 \ + CALC_48 \ + CALC_49 \ + CALC_50 \ + CALC_51 \ + CALC_52 \ + CALC_53 \ + CALC_54 \ + CALC_55 \ + CALC_56 \ + CALC_57 \ + CALC_58 \ + CALC_59 \ + ADDQ $128, R10 \ // move to next even-64-byte block + CMPQ R10, R11 \ // is current block the last one? + CMOVQCC R8, R10 \ // signal the last iteration smartly + CALC_60 \ + CALC_61 \ + CALC_62 \ + CALC_63 \ + CALC_64 \ + CALC_65 \ + CALC_66 \ + CALC_67 \ + CALC_68 \ + CALC_69 \ + CALC_70 \ + CALC_71 \ + CALC_72 \ + CALC_73 \ + CALC_74 \ + CALC_75 \ + CALC_76 \ + CALC_77 \ + CALC_78 \ + CALC_79 \ + UPDATE_HASH(AX,DX,BX,SI,DI) \ + CMPQ R10, R8 \ // is current block the last one? + JE loop\ + MOVL DX, CX \ + CALC_80 \ + CALC_81 \ + CALC_82 \ + CALC_83 \ + CALC_84 \ + CALC_85 \ + CALC_86 \ + CALC_87 \ + CALC_88 \ + CALC_89 \ + CALC_90 \ + CALC_91 \ + CALC_92 \ + CALC_93 \ + CALC_94 \ + CALC_95 \ + CALC_96 \ + CALC_97 \ + CALC_98 \ + CALC_99 \ + CALC_100 \ + CALC_101 \ + CALC_102 \ + CALC_103 \ + CALC_104 \ + CALC_105 \ + CALC_106 \ + CALC_107 \ + CALC_108 \ + CALC_109 \ + CALC_110 \ + CALC_111 \ + CALC_112 \ + CALC_113 \ + CALC_114 \ + CALC_115 \ + CALC_116 \ + CALC_117 \ + CALC_118 \ + CALC_119 \ + CALC_120 \ + CALC_121 \ + CALC_122 \ + CALC_123 \ + CALC_124 \ + CALC_125 \ + CALC_126 \ + CALC_127 \ + CALC_128 \ + CALC_129 \ + CALC_130 \ + CALC_131 \ + CALC_132 \ + CALC_133 \ + CALC_134 \ + CALC_135 \ + CALC_136 \ + CALC_137 \ + CALC_138 \ + CALC_139 \ + ADDQ $128, R13 \ //move to next even-64-byte block + CMPQ R13, R11 \ //is current block the last one? + CMOVQCC R8, R10 \ + CALC_140 \ + CALC_141 \ + CALC_142 \ + CALC_143 \ + CALC_144 \ + CALC_145 \ + CALC_146 \ + CALC_147 \ + CALC_148 \ + CALC_149 \ + CALC_150 \ + CALC_151 \ + CALC_152 \ + CALC_153 \ + CALC_154 \ + CALC_155 \ + CALC_156 \ + CALC_157 \ + CALC_158 \ + CALC_159 \ + UPDATE_HASH(SI,DI,DX,CX,BX) \ + MOVL SI, R12 \ //Reset state for AVX2 reg permutation + MOVL DI, SI \ + MOVL DX, DI \ + MOVL BX, DX \ + MOVL CX, AX \ + MOVL R12, CX \ + XCHGQ R15, R14 \ + JMP loop + + + +TEXT ·blockAVX2(SB),$1408-32 + + MOVQ dig+0(FP), DI + MOVQ p_base+8(FP), SI + MOVQ p_len+16(FP), DX + SHRQ $6, DX + SHLQ $6, DX + + MOVQ $K_XMM_AR<>(SB), R8 + + MOVQ DI, R9 + MOVQ SI, R10 + LEAQ 64(SI), R13 + + ADDQ SI, DX + ADDQ $64, DX + MOVQ DX, R11 + + CMPQ R13, R11 + CMOVQCC R8, R13 + + VMOVDQU BSWAP_SHUFB_CTL<>(SB), Y10 + + CALC // RET is inside macros + +DATA K_XMM_AR<>+0x00(SB)/4,$0x5a827999 +DATA K_XMM_AR<>+0x04(SB)/4,$0x5a827999 +DATA K_XMM_AR<>+0x08(SB)/4,$0x5a827999 +DATA K_XMM_AR<>+0x0c(SB)/4,$0x5a827999 +DATA K_XMM_AR<>+0x10(SB)/4,$0x5a827999 +DATA K_XMM_AR<>+0x14(SB)/4,$0x5a827999 +DATA K_XMM_AR<>+0x18(SB)/4,$0x5a827999 +DATA K_XMM_AR<>+0x1c(SB)/4,$0x5a827999 +DATA K_XMM_AR<>+0x20(SB)/4,$0x6ed9eba1 +DATA K_XMM_AR<>+0x24(SB)/4,$0x6ed9eba1 +DATA K_XMM_AR<>+0x28(SB)/4,$0x6ed9eba1 +DATA K_XMM_AR<>+0x2c(SB)/4,$0x6ed9eba1 +DATA K_XMM_AR<>+0x30(SB)/4,$0x6ed9eba1 +DATA K_XMM_AR<>+0x34(SB)/4,$0x6ed9eba1 +DATA K_XMM_AR<>+0x38(SB)/4,$0x6ed9eba1 +DATA K_XMM_AR<>+0x3c(SB)/4,$0x6ed9eba1 +DATA K_XMM_AR<>+0x40(SB)/4,$0x8f1bbcdc +DATA K_XMM_AR<>+0x44(SB)/4,$0x8f1bbcdc +DATA K_XMM_AR<>+0x48(SB)/4,$0x8f1bbcdc +DATA K_XMM_AR<>+0x4c(SB)/4,$0x8f1bbcdc +DATA K_XMM_AR<>+0x50(SB)/4,$0x8f1bbcdc +DATA K_XMM_AR<>+0x54(SB)/4,$0x8f1bbcdc +DATA K_XMM_AR<>+0x58(SB)/4,$0x8f1bbcdc +DATA K_XMM_AR<>+0x5c(SB)/4,$0x8f1bbcdc +DATA K_XMM_AR<>+0x60(SB)/4,$0xca62c1d6 +DATA K_XMM_AR<>+0x64(SB)/4,$0xca62c1d6 +DATA K_XMM_AR<>+0x68(SB)/4,$0xca62c1d6 +DATA K_XMM_AR<>+0x6c(SB)/4,$0xca62c1d6 +DATA K_XMM_AR<>+0x70(SB)/4,$0xca62c1d6 +DATA K_XMM_AR<>+0x74(SB)/4,$0xca62c1d6 +DATA K_XMM_AR<>+0x78(SB)/4,$0xca62c1d6 +DATA K_XMM_AR<>+0x7c(SB)/4,$0xca62c1d6 +GLOBL K_XMM_AR<>(SB),RODATA,$128 + +DATA BSWAP_SHUFB_CTL<>+0x00(SB)/4,$0x00010203 +DATA BSWAP_SHUFB_CTL<>+0x04(SB)/4,$0x04050607 +DATA BSWAP_SHUFB_CTL<>+0x08(SB)/4,$0x08090a0b +DATA BSWAP_SHUFB_CTL<>+0x0c(SB)/4,$0x0c0d0e0f +DATA BSWAP_SHUFB_CTL<>+0x10(SB)/4,$0x00010203 +DATA BSWAP_SHUFB_CTL<>+0x14(SB)/4,$0x04050607 +DATA BSWAP_SHUFB_CTL<>+0x18(SB)/4,$0x08090a0b +DATA BSWAP_SHUFB_CTL<>+0x1c(SB)/4,$0x0c0d0e0f +GLOBL BSWAP_SHUFB_CTL<>(SB),RODATA,$32 |