/******************************************************************************* Copyright (c) 2009-2019, Intel Corporation Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of Intel Corporation nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. *******************************************************************************/ /*----------------------------------------------------------------------- * * An implementation of SNOW 3G, the core algorithm for the * 3GPP Confidentiality and Integrity algorithms. * *-----------------------------------------------------------------------*/ #ifndef SNOW3G_COMMON_H #define SNOW3G_COMMON_H #include #include #include #include "intel-ipsec-mb.h" #include "include/snow3g.h" #include "include/snow3g_internal.h" #include "clear_regs_mem.h" #define CLEAR_MEM clear_mem #define CLEAR_VAR clear_var /* ------------------------------------------------------------------- * LFSR array shift by 1 position, 4 packets at a time * ------------------------------------------------------------------ */ #ifdef AVX2 /* LFSR array shift */ static inline void ShiftLFSR_8(snow3gKeyState8_t *pCtx) { pCtx->iLFSR_X = (pCtx->iLFSR_X + 1) & 15; } #endif /* AVX2 */ /* LFSR array shift */ static inline void ShiftLFSR_4(snow3gKeyState4_t *pCtx) { pCtx->iLFSR_X = (pCtx->iLFSR_X + 1) % 16; } /*--------------------------------------------------------- * @description * Gf2 modular multiplication/reduction * *---------------------------------------------------------*/ static inline uint64_t multiply_and_reduce64(uint64_t a, uint64_t b) { uint64_t msk; uint64_t res = 0; uint64_t i = 64; while (i--) { msk = ((int64_t)res >> 63) & 0x1b; res <<= 1; res ^= msk; msk = ((int64_t)b >> 63) & a; b <<= 1; res ^= msk; } return res; } #ifdef AVX2 /* ------------------------------------------------------------------- * ClockLFSR sub-function as defined in snow3g standard * S = LFSR[2] * ^ table_Alpha_div[LFSR[11] & 0xff] * ^ table_Alpha_mul[LFSR[0] & 0xff] * ------------------------------------------------------------------ */ static void C0_C11_8(__m256i *S, const __m256i *L0, const __m256i *L11) { __m256i mask, Sx, B11, B0, offset; offset = _mm256_set1_epi32(3); mask = _mm256_setr_epi32(0xF0F0F000, 0xF0F0F004, 0xF0F0F008, 0xF0F0F00C, 0xF0F0F000, 0xF0F0F004, 0xF0F0F008, 0xF0F0F00C); B11 = _mm256_shuffle_epi8(*L11, mask); *S = _mm256_i32gather_epi32(snow3g_table_A_div, B11, 4); mask = _mm256_add_epi32(mask, offset); B0 = _mm256_shuffle_epi8(*L0, mask); Sx = _mm256_i32gather_epi32(snow3g_table_A_mul, B0, 4); *S = _mm256_xor_si256(*S, Sx); } #endif /* AVX2 */ /* ------------------------------------------------------------------- * ClockLFSR sub-function as defined in snow3g standard * S = LFSR[2] * ^ table_Alpha_div[LFSR[11] & 0xff] * ^ table_Alpha_mul[LFSR[0] & 0xff] * ------------------------------------------------------------------ */ static inline void C0_C11_4(uint32_t *S, const __m128i *L0, const __m128i *L11) { unsigned B11[4], B0[4]; B11[0] = _mm_extract_epi8(*L11, 0); B11[1] = _mm_extract_epi8(*L11, 4); B11[2] = _mm_extract_epi8(*L11, 8); B11[3] = _mm_extract_epi8(*L11, 12); S[0] = snow3g_table_A_div[B11[0]]; S[1] = snow3g_table_A_div[B11[1]]; S[2] = snow3g_table_A_div[B11[2]]; S[3] = snow3g_table_A_div[B11[3]]; B0[0] = _mm_extract_epi8(*L0, 3); B0[1] = _mm_extract_epi8(*L0, 7); B0[2] = _mm_extract_epi8(*L0, 11); B0[3] = _mm_extract_epi8(*L0, 15); S[0] ^= snow3g_table_A_mul[B0[0]]; S[1] ^= snow3g_table_A_mul[B0[1]]; S[2] ^= snow3g_table_A_mul[B0[2]]; S[3] ^= snow3g_table_A_mul[B0[3]]; } #ifdef AVX2 /* ------------------------------------------------------------------- * ClockLFSR function as defined in snow3g standard * S = table_Alpha_div[LFSR[11] & 0xff] * ^ table_Alpha_mul[LFSR[0] >> 24] * ^ LFSR[2] ^ LFSR[0] << 8 ^ LFSR[11] >> 8 * ------------------------------------------------------------------ */ static inline void ClockLFSR_8(snow3gKeyState8_t *pCtx) { __m256i X2; __m256i S, T, U; U = pCtx->LFSR_X[pCtx->iLFSR_X]; S = pCtx->LFSR_X[(pCtx->iLFSR_X + 11) % 16]; C0_C11_8(&X2, &U, &S); T = _mm256_slli_epi32(U, 8); S = _mm256_srli_epi32(S, 8); U = _mm256_xor_si256(T, pCtx->LFSR_X[(pCtx->iLFSR_X + 2) % 16]); ShiftLFSR_8(pCtx); S = _mm256_xor_si256(S, U); S = _mm256_xor_si256(S, X2); pCtx->LFSR_X[(pCtx->iLFSR_X + 15) % 16] = S; } #endif /* AVX2 */ /* ------------------------------------------------------------------- * ClockLFSR function as defined in snow3g standard * S = table_Alpha_div[LFSR[11] & 0xff] * ^ table_Alpha_mul[LFSR[0] >> 24] * ^ LFSR[2] ^ LFSR[0] << 8 ^ LFSR[11] >> 8 * ------------------------------------------------------------------ */ static inline void ClockLFSR_4(snow3gKeyState4_t *pCtx) { uint32_t X2[4]; __m128i S, T, U; U = pCtx->LFSR_X[pCtx->iLFSR_X]; S = pCtx->LFSR_X[(pCtx->iLFSR_X + 11) % 16]; C0_C11_4(X2, &U, &S); T = _mm_slli_epi32(U, 8); S = _mm_srli_epi32(S, 8); U = _mm_xor_si128(T, pCtx->LFSR_X[(pCtx->iLFSR_X + 2) % 16]); ShiftLFSR_4(pCtx); /* (SSE4) */ T = _mm_insert_epi32(T, X2[0], 0); T = _mm_insert_epi32(T, X2[1], 1); T = _mm_insert_epi32(T, X2[2], 2); T = _mm_insert_epi32(T, X2[3], 3); S = _mm_xor_si128(S, U); S = _mm_xor_si128(S, T); pCtx->LFSR_X[(pCtx->iLFSR_X + 15) % 16] = S; } #ifdef AVX2 /* ------------------------------------------------------------------- * ClockFSM function as defined in snow3g standard * 8 packets at a time * ------------------------------------------------------------------ */ static inline void ClockFSM_8(snow3gKeyState8_t *pCtx, __m256i *data) { __m256i F, R, S2T0, S2T1, S2T2, S2T3, S1T0, S1T1, S1T2, S1T3; __m256i w3, w2, w1, w0, offset, mask; F = _mm256_add_epi32(pCtx->LFSR_X[(pCtx->iLFSR_X + 15)%16], pCtx->FSM_X[0]); R = _mm256_xor_si256(pCtx->LFSR_X[(pCtx->iLFSR_X + 5)%16], pCtx->FSM_X[2]); *data = _mm256_xor_si256(F, pCtx->FSM_X[1]); R = _mm256_add_epi32(R, pCtx->FSM_X[1]); offset = _mm256_set1_epi32(0x1); F = pCtx->FSM_X[1]; w3 = _mm256_setr_epi32(0xF0F0F000, 0xF0F0F004, 0xF0F0F008, 0xF0F0F00C, 0xF0F0F000, 0xF0F0F004, 0xF0F0F008, 0xF0F0F00C); mask = _mm256_shuffle_epi8(F,w3); S2T0 = _mm256_i32gather_epi32(S2_T0,mask,4); w2 = _mm256_add_epi32(w3,offset); mask = _mm256_shuffle_epi8(F,w2); S2T1 = _mm256_i32gather_epi32(S2_T1,mask,4); w1 = _mm256_add_epi32(w2,offset); mask = _mm256_shuffle_epi8(pCtx->FSM_X[1],w1); S2T2 = _mm256_i32gather_epi32(S2_T2,mask,4); w0 = _mm256_add_epi32(w1,offset); mask = _mm256_shuffle_epi8(F,w0); S2T3 = _mm256_i32gather_epi32(S2_T3,mask,4); F = pCtx->FSM_X[0]; w3 = _mm256_setr_epi32(0xF0F0F000, 0xF0F0F004, 0xF0F0F008, 0xF0F0F00C, 0xF0F0F010, 0xF0F0F014, 0xF0F0F018, 0xF0F0F01C); mask = _mm256_shuffle_epi8(F,w3); S1T0 = _mm256_i32gather_epi32(S1_T0,mask,4); w2 = _mm256_add_epi32(w3,offset); mask = _mm256_shuffle_epi8(F,w2); S1T1 = _mm256_i32gather_epi32(S1_T1,mask,4); w1 = _mm256_add_epi32(w2,offset); mask = _mm256_shuffle_epi8(F,w1); S1T2 = _mm256_i32gather_epi32(S1_T2,mask,4); w0 = _mm256_add_epi32(w1,offset); mask = _mm256_shuffle_epi8(F,w0); S1T3 = _mm256_i32gather_epi32(S1_T3,mask,4); S2T0 = _mm256_xor_si256(S2T0, S2T1); S2T2 = _mm256_xor_si256(S2T2, S2T3); S2T0 = _mm256_xor_si256(S2T0, S2T2); S1T0 = _mm256_xor_si256(S1T0, S1T1); S1T2 = _mm256_xor_si256(S1T2, S1T3); S1T0 = _mm256_xor_si256(S1T0, S1T2); pCtx->FSM_X[2] = S2T0; pCtx->FSM_X[1] = S1T0; pCtx->FSM_X[2] = S2T0; pCtx->FSM_X[0] = R; } #endif /* AVX2 */ /* ------------------------------------------------------------------- * ClockFSM function as defined in snow3g standard * 4 packets at a time * ------------------------------------------------------------------ */ static inline void ClockFSM_4(snow3gKeyState4_t *pCtx, __m128i *data) { __m128i F, R; #ifdef _WIN32 #pragma warning(push) #pragma warning(disable:4556) #endif #if defined (NO_AESNI) || defined (SAFE_LOOKUP) uint32_t L = 0; #endif uint32_t K = 0; F = _mm_add_epi32(pCtx->LFSR_X[(pCtx->iLFSR_X + 15) % 16], pCtx->FSM_X[0]); R = _mm_xor_si128(pCtx->LFSR_X[(pCtx->iLFSR_X + 5) % 16], pCtx->FSM_X[2]); *data = _mm_xor_si128(F, pCtx->FSM_X[1]); R = _mm_add_epi32(R, pCtx->FSM_X[1]); #if defined (NO_AESNI) || defined (SAFE_LOOKUP) S1_S2_4(pCtx->FSM_X[2], pCtx->FSM_X[1], pCtx->FSM_X[0], K, L, 0); S1_S2_4(pCtx->FSM_X[2], pCtx->FSM_X[1], pCtx->FSM_X[0], K, L, 1); S1_S2_4(pCtx->FSM_X[2], pCtx->FSM_X[1], pCtx->FSM_X[0], K, L, 2); S1_S2_4(pCtx->FSM_X[2], pCtx->FSM_X[1], pCtx->FSM_X[0], K, L, 3); #else S1_S2_4(pCtx->FSM_X[2], pCtx->FSM_X[1], pCtx->FSM_X[0], K, 0); S1_S2_4(pCtx->FSM_X[2], pCtx->FSM_X[1], pCtx->FSM_X[0], K, 1); S1_S2_4(pCtx->FSM_X[2], pCtx->FSM_X[1], pCtx->FSM_X[0], K, 2); S1_S2_4(pCtx->FSM_X[2], pCtx->FSM_X[1], pCtx->FSM_X[0], K, 3); #endif /* NO_AESNI */ pCtx->FSM_X[0] = R; #ifdef _WIN32 #pragma warning(pop) #endif } /** ******************************************************************************* * @description * This function generates 4 bytes of keystream 1 buffer at a time * * @param[in] pCtx Context where the scheduled keys are stored * @param[in/out] pKeyStream Pointer to generated keystream * *******************************************************************************/ static inline void snow3g_keystream_1_4(snow3gKeyState1_t *pCtx, uint32_t *pKeyStream) { uint32_t F; ClockFSM_1(pCtx, &F); *pKeyStream = F ^ pCtx->LFSR_S[0]; ClockLFSR_1(pCtx); } /** ******************************************************************************* * @description * This function generates 8 bytes of keystream 1 buffer at a time * * @param[in] pCtx Context where the scheduled keys are stored * @param[in/out] pKeyStream Pointer to generated keystream * *******************************************************************************/ static inline void snow3g_keystream_1_8(snow3gKeyState1_t *pCtx, uint64_t *pKeyStream) { uint64_t F; uint32_t FSM4; uint32_t V0, V1; uint32_t F0, F1; uint32_t R0, R1; uint32_t L0, L1, L11, L12; /* Merged clock FSM + clock LFSR + clock FSM + clockLFSR * in order to avoid redundancies in function processing * and less instruction immediate dependencies */ L0 = pCtx->LFSR_S[0]; V0 = pCtx->LFSR_S[2]; L1 = pCtx->LFSR_S[1]; V1 = pCtx->LFSR_S[3]; R1 = pCtx->FSM_R1; L11 = pCtx->LFSR_S[11]; L12 = pCtx->LFSR_S[12]; V0 ^= snow3g_table_A_mul[L0 >> 24]; V1 ^= snow3g_table_A_mul[L1 >> 24]; V0 ^= snow3g_table_A_div[L11 & 0xff]; V1 ^= snow3g_table_A_div[L12 & 0xff]; V0 ^= L0 << 8; V1 ^= L1 << 8; V0 ^= L11 >> 8; V1 ^= L12 >> 8; F0 = pCtx->LFSR_S[15] + R1; F0 ^= L0; F0 ^= pCtx->FSM_R2; R0 = pCtx->FSM_R3 ^ pCtx->LFSR_S[5]; R0 += pCtx->FSM_R2; S1_S2_S3_1(pCtx->FSM_R3, pCtx->FSM_R2, R1, FSM4, R0); R1 = pCtx->FSM_R3 ^ pCtx->LFSR_S[6]; F1 = V0 + R0; F1 ^= L1; F1 ^= pCtx->FSM_R2; R1 += pCtx->FSM_R2; pCtx->FSM_R3 = Snow3g_S2(pCtx->FSM_R2); pCtx->FSM_R2 = FSM4; pCtx->FSM_R1 = R1; /* Shift LFSR twice */ ShiftTwiceLFSR_1(pCtx); /* keystream mode LFSR update */ pCtx->LFSR_S[14] = V0; pCtx->LFSR_S[15] = V1; F = F0; F <<= 32; F |= (uint64_t)F1; *pKeyStream = F; } #ifdef AVX2 /** ******************************************************************************* * @description * This function generates 8 bytes of keystream 8 buffers at a time * * @param[in] pCtx Context where the scheduled keys are stored * @param[in/out] pKeyStream Pointer to generated keystream * *******************************************************************************/ static inline void snow3g_keystream_8_8(snow3gKeyState8_t *pCtx, __m256i *pKeyStreamLo, __m256i *pKeyStreamHi) { __m256i H, L; /* first set of 4 bytes */ ClockFSM_8(pCtx, &L); L = _mm256_xor_si256(L, pCtx->LFSR_X[pCtx->iLFSR_X]); ClockLFSR_8(pCtx); /* second set of 4 bytes */ ClockFSM_8(pCtx, &H); H = _mm256_xor_si256(H, pCtx->LFSR_X[pCtx->iLFSR_X]); ClockLFSR_8(pCtx); /* merge the 2 sets */ *pKeyStreamLo = _mm256_unpacklo_epi32(H, L); *pKeyStreamHi = _mm256_unpackhi_epi32(H, L); } /** ******************************************************************************* * @description * This function generates 4 bytes of keystream 8 buffers at a time * * @param[in] pCtx Context where the scheduled keys are stored * @param[in/out] pKeyStream Pointer to generated keystream * *******************************************************************************/ static inline void snow3g_keystream_8_4(snow3gKeyState8_t *pCtx, __m256i *pKeyStream) { __m256i F; ClockFSM_8(pCtx, &F); *pKeyStream = _mm256_xor_si256(F, pCtx->LFSR_X[pCtx->iLFSR_X]); ClockLFSR_8(pCtx); } /** ***************************************************************************** * @description * This function generates 32 bytes of keystream 8 buffers at a time * * @param[in] pCtx Context where the scheduled keys are stored * @param[in/out] pKeyStream Array of generated keystreams * ******************************************************************************/ static inline void snow3g_keystream_8_32(snow3gKeyState8_t *pCtx, __m256i *pKeyStream) { __m256i temp[8]; /** produces the next 4 bytes for each buffer */ int i; /** Byte reversal on each KS */ __m256i mask1 = {0x0001020304050607ULL, 0x08090a0b0c0d0e0fULL, 0x0001020304050607ULL, 0x08090a0b0c0d0e0fULL}; /** Reversal, shifted 4 bytes right */ __m256i mask2 = {0x0405060708090a0bULL, 0x0c0d0e0f00010203ULL, 0x0405060708090a0bULL, 0x0c0d0e0f00010203ULL}; /** Reversal, shifted 8 bytes right */ __m256i mask3 = {0x08090a0b0c0d0e0fULL, 0x0001020304050607ULL, 0x08090a0b0c0d0e0fULL, 0x0001020304050607ULL}; /** Reversal, shifted 12 bytes right */ __m256i mask4 = {0x0c0d0e0f00010203ULL, 0x0405060708090a0bULL, 0x0c0d0e0f00010203ULL, 0x0405060708090a0bULL}; snow3g_keystream_8_4(pCtx, &temp[0]); snow3g_keystream_8_4(pCtx, &temp[1]); snow3g_keystream_8_4(pCtx, &temp[2]); snow3g_keystream_8_4(pCtx, &temp[3]); snow3g_keystream_8_4(pCtx, &temp[4]); snow3g_keystream_8_4(pCtx, &temp[5]); snow3g_keystream_8_4(pCtx, &temp[6]); snow3g_keystream_8_4(pCtx, &temp[7]); temp[0] = _mm256_shuffle_epi8(temp[0], mask1); temp[1] = _mm256_shuffle_epi8(temp[1], mask2); temp[2] = _mm256_shuffle_epi8(temp[2], mask3); temp[3] = _mm256_shuffle_epi8(temp[3], mask4); temp[4] = _mm256_shuffle_epi8(temp[4], mask1); temp[5] = _mm256_shuffle_epi8(temp[5], mask2); temp[6] = _mm256_shuffle_epi8(temp[6], mask3); temp[7] = _mm256_shuffle_epi8(temp[7], mask4); __m256i blended[8]; /* blends KS together: 128bit slice consists of 4 32-bit words for one packet */ blended[0] = _mm256_blend_epi32(temp[0], temp[1], 0xaa); blended[1] = _mm256_blend_epi32(temp[0], temp[1], 0x55); blended[2] = _mm256_blend_epi32(temp[2], temp[3], 0xaa); blended[3] = _mm256_blend_epi32(temp[2], temp[3], 0x55); blended[4] = _mm256_blend_epi32(temp[4], temp[5], 0xaa); blended[5] = _mm256_blend_epi32(temp[4], temp[5], 0x55); blended[6] = _mm256_blend_epi32(temp[6], temp[7], 0xaa); blended[7] = _mm256_blend_epi32(temp[6], temp[7], 0x55); temp[0] = _mm256_blend_epi32(blended[0], blended[2], 0xcc); temp[1] = _mm256_blend_epi32(blended[1], blended[3], 0x99); temp[2] = _mm256_blend_epi32(blended[0], blended[2], 0x33); temp[3] = _mm256_blend_epi32(blended[1], blended[3], 0x66); temp[4] = _mm256_blend_epi32(blended[4], blended[6], 0xcc); temp[5] = _mm256_blend_epi32(blended[5], blended[7], 0x99); temp[6] = _mm256_blend_epi32(blended[4], blended[6], 0x33); temp[7] = _mm256_blend_epi32(blended[5], blended[7], 0x66); /** sorts 32 bit words back into order */ blended[0] = temp[0]; blended[1] = _mm256_shuffle_epi32(temp[1], 0x39); blended[2] = _mm256_shuffle_epi32(temp[2], 0x4e); blended[3] = _mm256_shuffle_epi32(temp[3], 0x93); blended[4] = temp[4]; blended[5] = _mm256_shuffle_epi32(temp[5], 0x39); blended[6] = _mm256_shuffle_epi32(temp[6], 0x4e); blended[7] = _mm256_shuffle_epi32(temp[7], 0x93); for (i = 0; i < 4; i++) { pKeyStream[i] = _mm256_permute2x128_si256(blended[i], blended[i + 4], 0x20); pKeyStream[i + 4] = _mm256_permute2x128_si256( blended[i], blended[i + 4], 0x31); } } #endif /* AVX2 */ /** ******************************************************************************* * @description * This function generates 4 bytes of keystream 4 buffers at a time * * @param[in] pCtx Context where the scheduled keys are stored * @param[in/out] pKeyStream Pointer to generated keystream * *******************************************************************************/ static inline void snow3g_keystream_4_4(snow3gKeyState4_t *pCtx, __m128i *pKeyStream) { __m128i F; ClockFSM_4(pCtx, &F); *pKeyStream = _mm_xor_si128(F, pCtx->LFSR_X[pCtx->iLFSR_X]); ClockLFSR_4(pCtx); } /** ******************************************************************************* * @description * This function generates 8 bytes of keystream 4 buffers at a time * * @param[in] pCtx Context where the scheduled keys are stored * @param[in/out] pKeyStreamLo Pointer to lower end of generated keystream * @param[in/out] pKeyStreamHi Pointer to higer end of generated keystream * *******************************************************************************/ static inline void snow3g_keystream_4_8(snow3gKeyState4_t *pCtx, __m128i *pKeyStreamLo, __m128i *pKeyStreamHi) { __m128i H, L; /* first set of 4 bytes */ ClockFSM_4(pCtx, &L); L = _mm_xor_si128(L, pCtx->LFSR_X[pCtx->iLFSR_X]); ClockLFSR_4(pCtx); /* second set of 4 bytes */ ClockFSM_4(pCtx, &H); H = _mm_xor_si128(H, pCtx->LFSR_X[pCtx->iLFSR_X]); ClockLFSR_4(pCtx); /* merge the 2 sets */ *pKeyStreamLo = _mm_unpacklo_epi32(H, L); *pKeyStreamHi = _mm_unpackhi_epi32(H, L); } /** ******************************************************************************* * @description * This function initializes the key schedule for 4 buffers for snow3g f8/f9. * * @param [in] pCtx Context where the scheduled keys are stored * @param [in] pKeySched Key schedule * @param [in] pIV1 IV for buffer 1 * @param [in] pIV2 IV for buffer 2 * @param [in] pIV3 IV for buffer 3 * @param [in] pIV4 IV for buffer 4 * *******************************************************************************/ static inline void snow3gStateInitialize_4(snow3gKeyState4_t *pCtx, const snow3g_key_schedule_t *pKeySched, const void *pIV1, const void *pIV2, const void *pIV3, const void *pIV4) { uint32_t K, L; int i; __m128i R, S, T, U; __m128i V0, V1, T0, T1; /* Initialize the LFSR table from constants, Keys, and IV */ /* Load complete 128b IV into register (SSE2)*/ uint64_t sm[2] = {0x0405060700010203ULL, 0x0c0d0e0f08090a0bULL}; __m128i *swapMask = (__m128i *) sm; R = _mm_loadu_si128((const __m128i *)pIV1); S = _mm_loadu_si128((const __m128i *)pIV2); T = _mm_loadu_si128((const __m128i *)pIV3); U = _mm_loadu_si128((const __m128i *)pIV4); /* initialize the array block (SSE4) */ for (i = 0; i < 4; i++) { K = pKeySched->k[i]; L = ~K; V0 = _mm_set1_epi32(K); V1 = _mm_set1_epi32(L); pCtx->LFSR_X[i + 4] = V0; pCtx->LFSR_X[i + 12] = V0; pCtx->LFSR_X[i + 0] = V1; pCtx->LFSR_X[i + 8] = V1; } /* Update the schedule structure with IVs */ /* Store the 4 IVs in LFSR by a column/row matrix swap * after endianness correction */ /* endianness swap (SSSE3) */ R = _mm_shuffle_epi8(R, *swapMask); S = _mm_shuffle_epi8(S, *swapMask); T = _mm_shuffle_epi8(T, *swapMask); U = _mm_shuffle_epi8(U, *swapMask); /* row/column dword inversion (SSE2) */ T0 = _mm_unpacklo_epi32(R, S); R = _mm_unpackhi_epi32(R, S); T1 = _mm_unpacklo_epi32(T, U); T = _mm_unpackhi_epi32(T, U); /* row/column qword inversion (SSE2) */ U = _mm_unpackhi_epi64(R, T); T = _mm_unpacklo_epi64(R, T); S = _mm_unpackhi_epi64(T0, T1); R = _mm_unpacklo_epi64(T0, T1); /*IV ^ LFSR (SSE2) */ pCtx->LFSR_X[15] = _mm_xor_si128(pCtx->LFSR_X[15], U); pCtx->LFSR_X[12] = _mm_xor_si128(pCtx->LFSR_X[12], T); pCtx->LFSR_X[10] = _mm_xor_si128(pCtx->LFSR_X[10], S); pCtx->LFSR_X[9] = _mm_xor_si128(pCtx->LFSR_X[9], R); pCtx->iLFSR_X = 0; /* FSM initialization (SSE2) */ S = _mm_setzero_si128(); for (i = 0; i < 3; i++) pCtx->FSM_X[i] = S; /* Initialisation rounds */ for (i = 0; i < 32; i++) { ClockFSM_4(pCtx, &S); ClockLFSR_4(pCtx); pCtx->LFSR_X[(pCtx->iLFSR_X + 15) % 16] = _mm_xor_si128( pCtx->LFSR_X[(pCtx->iLFSR_X + 15) % 16], S); } } #ifdef AVX2 /** ******************************************************************************* * @description * This function intializes the key schedule for 8 buffers with * individual keys, for snow3g f8/f9. * * @param [in] pCtx Context where scheduled keys are stored * @param [in] pKeySched Key schedule * @param [in] pIV1 IV for buffer 1 * @param [in] pIV2 IV for buffer 2 * @param [in] pIV3 IV for buffer 3 * @param [in] pIV4 IV for buffer 4 * @param [in] pIV5 IV for buffer 5 * @param [in] pIV6 IV for buffer 6 * @param [in] pIV7 IV for buffer 7 * @param [in] pIV8 IV for buffer 8 * *******************************************************************************/ static inline void snow3gStateInitialize_8_multiKey(snow3gKeyState8_t *pCtx, const snow3g_key_schedule_t * const KeySched[], const void * const pIV[]) { DECLARE_ALIGNED(uint32_t k[8], 32); DECLARE_ALIGNED(uint32_t l[8], 32); __m256i *K = (__m256i *)k; __m256i *L = (__m256i *)l; int i, j; __m256i mR, mS, mT, mU, T0, T1; /* Initialize the LFSR table from constants, Keys, and IV */ /* Load complete 256b IV into register (SSE2)*/ __m256i swapMask = {0x0405060700010203ULL, 0x0c0d0e0f08090a0bULL, 0x0405060700010203ULL, 0x0c0d0e0f08090a0bULL}; mR = _mm256_loadu2_m128i((const __m128i *)pIV[4], (const __m128i *)pIV[0]); mS = _mm256_loadu2_m128i((const __m128i *)pIV[5], (const __m128i *)pIV[1]); mT = _mm256_loadu2_m128i((const __m128i *)pIV[6], (const __m128i *)pIV[2]); mU = _mm256_loadu2_m128i((const __m128i *)pIV[7], (const __m128i *)pIV[3]); /* initialize the array block (SSE4) */ for (i = 0; i < 4; i++) { for (j = 0; j < 8; j++) { k[j] = KeySched[j]->k[i]; l[j] = ~k[j]; } pCtx->LFSR_X[i + 4] = *K; pCtx->LFSR_X[i + 12] = *K; pCtx->LFSR_X[i + 0] = *L; pCtx->LFSR_X[i + 8] = *L; } /* Update the schedule structure with IVs */ /* Store the 4 IVs in LFSR by a column/row matrix swap * after endianness correction */ /* endianness swap (SSSE3) */ mR = _mm256_shuffle_epi8(mR, swapMask); mS = _mm256_shuffle_epi8(mS, swapMask); mT = _mm256_shuffle_epi8(mT, swapMask); mU = _mm256_shuffle_epi8(mU, swapMask); /* row/column dword inversion (SSE2) */ T0 = _mm256_unpacklo_epi32(mR, mS); mR = _mm256_unpackhi_epi32(mR, mS); T1 = _mm256_unpacklo_epi32(mT, mU); mT = _mm256_unpackhi_epi32(mT, mU); /* row/column qword inversion (SSE2) */ mU = _mm256_unpackhi_epi64(mR, mT); mT = _mm256_unpacklo_epi64(mR, mT); mS = _mm256_unpackhi_epi64(T0, T1); mR = _mm256_unpacklo_epi64(T0, T1); /*IV ^ LFSR (SSE2) */ pCtx->LFSR_X[15] = _mm256_xor_si256(pCtx->LFSR_X[15], mU); pCtx->LFSR_X[12] = _mm256_xor_si256(pCtx->LFSR_X[12], mT); pCtx->LFSR_X[10] = _mm256_xor_si256(pCtx->LFSR_X[10], mS); pCtx->LFSR_X[9] = _mm256_xor_si256(pCtx->LFSR_X[9], mR); pCtx->iLFSR_X = 0; /* FSM initialization (SSE2) */ mS = _mm256_setzero_si256(); for (i = 0; i < 3; i++) pCtx->FSM_X[i] = mS; /* Initialisation rounds */ for (i = 0; i < 32; i++) { ClockFSM_8(pCtx, &mS); ClockLFSR_8(pCtx); pCtx->LFSR_X[(pCtx->iLFSR_X + 15) % 16] = _mm256_xor_si256( pCtx->LFSR_X[(pCtx->iLFSR_X + 15) % 16], mS); } } /** ******************************************************************************* * @description * This function initializes the key schedule for 8 buffers for snow3g f8/f9. * * @param [in] pCtx Context where the scheduled keys are stored * @param [in] pKeySched Key schedule * @param [in] pIV1 IV for buffer 1 * @param [in] pIV2 IV for buffer 2 * @param [in] pIV3 IV for buffer 3 * @param [in] pIV4 IV for buffer 4 * @param [in] pIV5 IV for buffer 5 * @param [in] pIV6 IV for buffer 6 * @param [in] pIV7 IV for buffer 7 * @param [in] pIV8 IV for buffer 8 * *******************************************************************************/ static inline void snow3gStateInitialize_8(snow3gKeyState8_t *pCtx, const snow3g_key_schedule_t *pKeySched, const void *pIV1, const void *pIV2, const void *pIV3, const void *pIV4, const void *pIV5, const void *pIV6, const void *pIV7, const void *pIV8) { uint32_t K, L; int i; __m256i mR, mS, mT, mU, V0, V1, T0, T1; /* Initialize the LFSR table from constants, Keys, and IV */ /* Load complete 256b IV into register (SSE2)*/ __m256i swapMask = {0x0405060700010203ULL, 0x0c0d0e0f08090a0bULL, 0x0405060700010203ULL, 0x0c0d0e0f08090a0bULL}; mR = _mm256_loadu2_m128i((const __m128i *)pIV5, (const __m128i *)pIV1); mS = _mm256_loadu2_m128i((const __m128i *)pIV6, (const __m128i *)pIV2); mT = _mm256_loadu2_m128i((const __m128i *)pIV7, (const __m128i *)pIV3); mU = _mm256_loadu2_m128i((const __m128i *)pIV8, (const __m128i *)pIV4); /* initialize the array block (SSE4) */ for (i = 0; i < 4; i++) { K = pKeySched->k[i]; L = ~K; V0 = _mm256_set1_epi32(K); V1 = _mm256_set1_epi32(L); pCtx->LFSR_X[i + 4] = V0; pCtx->LFSR_X[i + 12] = V0; pCtx->LFSR_X[i + 0] = V1; pCtx->LFSR_X[i + 8] = V1; } /* Update the schedule structure with IVs */ /* Store the 4 IVs in LFSR by a column/row matrix swap * after endianness correction */ /* endianness swap (SSSE3) */ mR = _mm256_shuffle_epi8(mR, swapMask); mS = _mm256_shuffle_epi8(mS, swapMask); mT = _mm256_shuffle_epi8(mT, swapMask); mU = _mm256_shuffle_epi8(mU, swapMask); /* row/column dword inversion (SSE2) */ T0 = _mm256_unpacklo_epi32(mR, mS); mR = _mm256_unpackhi_epi32(mR, mS); T1 = _mm256_unpacklo_epi32(mT, mU); mT = _mm256_unpackhi_epi32(mT, mU); /* row/column qword inversion (SSE2) */ mU = _mm256_unpackhi_epi64(mR, mT); mT = _mm256_unpacklo_epi64(mR, mT); mS = _mm256_unpackhi_epi64(T0, T1); mR = _mm256_unpacklo_epi64(T0, T1); /*IV ^ LFSR (SSE2) */ pCtx->LFSR_X[15] = _mm256_xor_si256(pCtx->LFSR_X[15], mU); pCtx->LFSR_X[12] = _mm256_xor_si256(pCtx->LFSR_X[12], mT); pCtx->LFSR_X[10] = _mm256_xor_si256(pCtx->LFSR_X[10], mS); pCtx->LFSR_X[9] = _mm256_xor_si256(pCtx->LFSR_X[9], mR); pCtx->iLFSR_X = 0; /* FSM initialization (SSE2) */ mS = _mm256_setzero_si256(); for (i = 0; i < 3; i++) pCtx->FSM_X[i] = mS; /* Initialisation rounds */ for (i = 0; i < 32; i++) { ClockFSM_8(pCtx, &mS); ClockLFSR_8(pCtx); pCtx->LFSR_X[(pCtx->iLFSR_X + 15) % 16] = _mm256_xor_si256( pCtx->LFSR_X[(pCtx->iLFSR_X + 15) % 16], mS); } } #endif /* AVX2 */ static inline void preserve_bits(uint64_t *KS, const uint8_t *pcBufferOut, const uint8_t *pcBufferIn, SafeBuf *safeOutBuf, SafeBuf *safeInBuf, const uint8_t bit_len, const uint8_t byte_len) { const uint64_t mask = UINT64_MAX << (SNOW3G_BLOCK_SIZE * 8 - bit_len); /* Clear the last bits of the keystream and the input * (input only in out-of-place case) */ *KS &= mask; if (pcBufferIn != pcBufferOut) { const uint64_t swapMask = BSWAP64(mask); safeInBuf->b64 &= swapMask; /* * Merge the last bits from the output, to be preserved, * in the keystream, to be XOR'd with the input * (which last bits are 0, maintaining the output bits) */ memcpy_keystrm(safeOutBuf->b8, pcBufferOut, byte_len); *KS |= BSWAP64(safeOutBuf->b64 & ~swapMask); } } /** ******************************************************************************* * @description * This function is the core snow3g bit algorithm * for the 3GPP confidentiality algorithm * * @param[in] pCtx Context where the scheduled keys are stored * @param[in] pBufferIn Input buffer * @param[out] pBufferOut Output buffer * @param[in] cipherLengthInBits length in bits of the data to be encrypted * @param[in] bitOffset offset in input buffer, where data are valid * *******************************************************************************/ static inline void f8_snow3g_bit(snow3gKeyState1_t *pCtx, const void *pIn, void *pOut, const uint32_t lengthInBits, const uint32_t offsetInBits) { const uint8_t *pBufferIn = pIn; uint8_t *pBufferOut = pOut; uint32_t cipherLengthInBits = lengthInBits; uint64_t shiftrem = 0; uint64_t KS8, KS8bit; /* 8 bytes of keystream */ const uint8_t *pcBufferIn = pBufferIn + (offsetInBits / 8); uint8_t *pcBufferOut = pBufferOut + (offsetInBits / 8); /* Offset into the first byte (0 - 7 bits) */ uint32_t remainOffset = offsetInBits % 8; uint32_t byteLength = (cipherLengthInBits + 7) / 8; SafeBuf safeInBuf = {0}; SafeBuf safeOutBuf = {0}; /* Now run the block cipher */ /* Start with potential partial block (due to offset and length) */ snow3g_keystream_1_8(pCtx, &KS8); KS8bit = KS8 >> remainOffset; /* Only one block to encrypt */ if (cipherLengthInBits < (64 - remainOffset)) { byteLength = (cipherLengthInBits + 7) / 8; memcpy_keystrm(safeInBuf.b8, pcBufferIn, byteLength); /* * If operation is Out-of-place and there is offset * to be applied, "remainOffset" bits from the output buffer * need to be preserved (only applicable to first byte, * since remainOffset is up to 7 bits) */ if ((pIn != pOut) && remainOffset) { const uint8_t mask8 = (uint8_t) (1 << (8 - remainOffset)) - 1; safeInBuf.b8[0] = (safeInBuf.b8[0] & mask8) | (pcBufferOut[0] & ~mask8); } /* If last byte is a partial byte, the last bits of the output * need to be preserved */ const uint8_t bitlen_with_off = remainOffset + cipherLengthInBits; if ((bitlen_with_off & 0x7) != 0) preserve_bits(&KS8bit, pcBufferOut, pcBufferIn, &safeOutBuf, &safeInBuf, bitlen_with_off, byteLength); xor_keystrm_rev(safeOutBuf.b8, safeInBuf.b8, KS8bit); memcpy_keystrm(pcBufferOut, safeOutBuf.b8, byteLength); return; } /* * If operation is Out-of-place and there is offset * to be applied, "remainOffset" bits from the output buffer * need to be preserved (only applicable to first byte, * since remainOffset is up to 7 bits) */ if ((pIn != pOut) && remainOffset) { const uint8_t mask8 = (uint8_t)(1 << (8 - remainOffset)) - 1; memcpy_keystrm(safeInBuf.b8, pcBufferIn, 8); safeInBuf.b8[0] = (safeInBuf.b8[0] & mask8) | (pcBufferOut[0] & ~mask8); xor_keystrm_rev(pcBufferOut, safeInBuf.b8, KS8bit); pcBufferIn += SNOW3G_BLOCK_SIZE; } else { /* At least 64 bits to produce (including offset) */ pcBufferIn = xor_keystrm_rev(pcBufferOut, pcBufferIn, KS8bit); } if (remainOffset != 0) shiftrem = KS8 << (64 - remainOffset); cipherLengthInBits -= SNOW3G_BLOCK_SIZE * 8 - remainOffset; pcBufferOut += SNOW3G_BLOCK_SIZE; while (cipherLengthInBits) { /* produce the next block of keystream */ snow3g_keystream_1_8(pCtx, &KS8); KS8bit = (KS8 >> remainOffset) | shiftrem; if (remainOffset != 0) shiftrem = KS8 << (64 - remainOffset); if (cipherLengthInBits >= SNOW3G_BLOCK_SIZE * 8) { pcBufferIn = xor_keystrm_rev(pcBufferOut, pcBufferIn, KS8bit); cipherLengthInBits -= SNOW3G_BLOCK_SIZE * 8; pcBufferOut += SNOW3G_BLOCK_SIZE; /* loop variant */ } else { /* end of the loop, handle the last bytes */ byteLength = (cipherLengthInBits + 7) / 8; memcpy_keystrm(safeInBuf.b8, pcBufferIn, byteLength); /* If last byte is a partial byte, the last bits * of the output need to be preserved */ if ((cipherLengthInBits & 0x7) != 0) preserve_bits(&KS8bit, pcBufferOut, pcBufferIn, &safeOutBuf, &safeInBuf, cipherLengthInBits, byteLength); xor_keystrm_rev(safeOutBuf.b8, safeInBuf.b8, KS8bit); memcpy_keystrm(pcBufferOut, safeOutBuf.b8, byteLength); cipherLengthInBits = 0; } } #ifdef SAFE_DATA CLEAR_VAR(&KS8, sizeof(KS8)); CLEAR_VAR(&KS8bit, sizeof(KS8bit)); CLEAR_MEM(&safeInBuf, sizeof(safeInBuf)); CLEAR_MEM(&safeOutBuf, sizeof(safeOutBuf)); #endif } /** ******************************************************************************* * @description * This function is the core snow3g algorithm for * the 3GPP confidentiality and integrity algorithm. * * @param[in] pCtx Context where the scheduled keys are stored * @param[in] pBufferIn Input buffer * @param[out] pBufferOut Output buffer * @param[in] lengthInBytes length in bytes of the data to be encrypted * *******************************************************************************/ static inline void f8_snow3g(snow3gKeyState1_t *pCtx, const void *pIn, void *pOut, const uint32_t lengthInBytes) { uint32_t qwords = lengthInBytes / SNOW3G_8_BYTES; /* number of qwords */ uint32_t words = lengthInBytes & 4; /* remaining word if not 0 */ uint32_t bytes = lengthInBytes & 3; /* remaining bytes */ uint32_t KS4; /* 4 bytes of keystream */ uint64_t KS8; /* 8 bytes of keystream */ const uint8_t *pBufferIn = pIn; uint8_t *pBufferOut = pOut; /* process 64 bits at a time */ while (qwords--) { /* generate keystream 8 bytes at a time */ snow3g_keystream_1_8(pCtx, &KS8); /* xor keystream 8 bytes at a time */ pBufferIn = xor_keystrm_rev(pBufferOut, pBufferIn, KS8); pBufferOut += SNOW3G_8_BYTES; } /* check for remaining 0 to 7 bytes */ if (0 != words) { if (bytes) { /* 5 to 7 last bytes, process 8 bytes */ uint8_t buftemp[8]; uint8_t safeBuff[8]; memset(safeBuff, 0, SNOW3G_8_BYTES); snow3g_keystream_1_8(pCtx, &KS8); memcpy_keystrm(safeBuff, pBufferIn, 4 + bytes); xor_keystrm_rev(buftemp, safeBuff, KS8); memcpy_keystrm(pBufferOut, buftemp, 4 + bytes); #ifdef SAFE_DATA CLEAR_MEM(&safeBuff, sizeof(safeBuff)); CLEAR_MEM(&buftemp, sizeof(buftemp)); #endif } else { /* exactly 4 last bytes */ snow3g_keystream_1_4(pCtx, &KS4); xor_keystream_reverse_32(pBufferOut, pBufferIn, KS4); } } else if (0 != bytes) { /* 1 to 3 last bytes */ uint8_t buftemp[4]; uint8_t safeBuff[4]; memset(safeBuff, 0, SNOW3G_4_BYTES); snow3g_keystream_1_4(pCtx, &KS4); memcpy_keystream_32(safeBuff, pBufferIn, bytes); xor_keystream_reverse_32(buftemp, safeBuff, KS4); memcpy_keystream_32(pBufferOut, buftemp, bytes); #ifdef SAFE_DATA CLEAR_MEM(&safeBuff, sizeof(safeBuff)); CLEAR_MEM(&buftemp, sizeof(buftemp)); #endif } #ifdef SAFE_DATA CLEAR_VAR(&KS4, sizeof(KS4)); CLEAR_VAR(&KS8, sizeof(KS8)); #endif } #ifdef AVX2 /** ******************************************************************************* * @description * This function converts the state from a 4 buffer state structure to 1 * buffer state structure. * * @param[in] pSrcState Pointer to the source state * @param[in] pDstState Pointer to the destination state * @param[in] NumBuffers Number of buffers * *******************************************************************************/ static inline void snow3gStateConvert_8(snow3gKeyState8_t *pSrcState, snow3gKeyState1_t *pDstState, uint32_t NumBuffers) { uint32_t T = 0, iLFSR_X = pSrcState->iLFSR_X; __m256i *LFSR_X = pSrcState->LFSR_X; int i; for (i = 0; i < 16; i++) { switch (NumBuffers) { case 0: T = _mm256_extract_epi32(LFSR_X[(i + iLFSR_X) % 16], 0); break; case 1: T = _mm256_extract_epi32(LFSR_X[(i + iLFSR_X) % 16], 1); break; case 2: T = _mm256_extract_epi32(LFSR_X[(i + iLFSR_X) % 16], 2); break; case 3: T = _mm256_extract_epi32(LFSR_X[(i + iLFSR_X) % 16], 3); break; case 4: T = _mm256_extract_epi32(LFSR_X[(i + iLFSR_X) % 16], 4); break; case 5: T = _mm256_extract_epi32(LFSR_X[(i + iLFSR_X) % 16], 5); break; case 6: T = _mm256_extract_epi32(LFSR_X[(i + iLFSR_X) % 16], 6); break; case 7: T = _mm256_extract_epi32(LFSR_X[(i + iLFSR_X) % 16], 7); break; } pDstState->LFSR_S[i] = T; } i = 0; switch (NumBuffers) { case 0: T = _mm256_extract_epi32(pSrcState->FSM_X[i], 0); break; case 1: T = _mm256_extract_epi32(pSrcState->FSM_X[i], 1); break; case 2: T = _mm256_extract_epi32(pSrcState->FSM_X[i], 2); break; case 3: T = _mm256_extract_epi32(pSrcState->FSM_X[i], 3); break; case 4: T = _mm256_extract_epi32(pSrcState->FSM_X[i], 4); break; case 5: T = _mm256_extract_epi32(pSrcState->FSM_X[i], 5); break; case 6: T = _mm256_extract_epi32(pSrcState->FSM_X[i], 6); break; case 7: T = _mm256_extract_epi32(pSrcState->FSM_X[i], 7); break; } pDstState->FSM_R1 = T; i = 1; switch (NumBuffers) { case 0: T = _mm256_extract_epi32(pSrcState->FSM_X[i], 0); break; case 1: T = _mm256_extract_epi32(pSrcState->FSM_X[i], 1); break; case 2: T = _mm256_extract_epi32(pSrcState->FSM_X[i], 2); break; case 3: T = _mm256_extract_epi32(pSrcState->FSM_X[i], 3); break; case 4: T = _mm256_extract_epi32(pSrcState->FSM_X[i], 4); break; case 5: T = _mm256_extract_epi32(pSrcState->FSM_X[i], 5); break; case 6: T = _mm256_extract_epi32(pSrcState->FSM_X[i], 6); break; case 7: T = _mm256_extract_epi32(pSrcState->FSM_X[i], 7); break; } pDstState->FSM_R2 = T; i = 2; switch (NumBuffers) { case 0: T = _mm256_extract_epi32(pSrcState->FSM_X[i], 0); break; case 1: T = _mm256_extract_epi32(pSrcState->FSM_X[i], 1); break; case 2: T = _mm256_extract_epi32(pSrcState->FSM_X[i], 2); break; case 3: T = _mm256_extract_epi32(pSrcState->FSM_X[i], 3); break; case 4: T = _mm256_extract_epi32(pSrcState->FSM_X[i], 4); break; case 5: T = _mm256_extract_epi32(pSrcState->FSM_X[i], 5); break; case 6: T = _mm256_extract_epi32(pSrcState->FSM_X[i], 6); break; case 7: T = _mm256_extract_epi32(pSrcState->FSM_X[i], 7); break; } pDstState->FSM_R3 = T; } #endif /* AVX2 */ /** ******************************************************************************* * @description * This function converts the state from a 4 buffer state structure to 1 * buffer state structure. * * @param[in] pSrcState Pointer to the source state * @param[in] pDstState Pointer to the destination state * @param[in] NumBuffers Number of buffers * *******************************************************************************/ static inline void snow3gStateConvert_4(snow3gKeyState4_t *pSrcState, snow3gKeyState1_t *pDstState, uint32_t NumBuffers) { uint32_t i; uint32_t T = 0, iLFSR_X = pSrcState->iLFSR_X; __m128i *LFSR_X = pSrcState->LFSR_X; for (i = 0; i < 16; i++) { switch (NumBuffers) { case 0: T = _mm_extract_epi32(LFSR_X[(i + iLFSR_X) % 16], 0); break; case 1: T = _mm_extract_epi32(LFSR_X[(i + iLFSR_X) % 16], 1); break; case 2: T = _mm_extract_epi32(LFSR_X[(i + iLFSR_X) % 16], 2); break; case 3: T = _mm_extract_epi32(LFSR_X[(i + iLFSR_X) % 16], 3); break; } pDstState->LFSR_S[i] = T; } i = 0; switch (NumBuffers) { case 0: T = _mm_extract_epi32(pSrcState->FSM_X[i], 0); break; case 1: T = _mm_extract_epi32(pSrcState->FSM_X[i], 1); break; case 2: T = _mm_extract_epi32(pSrcState->FSM_X[i], 2); break; case 3: T = _mm_extract_epi32(pSrcState->FSM_X[i], 3); break; } pDstState->FSM_R1 = T; i = 1; switch (NumBuffers) { case 0: T = _mm_extract_epi32(pSrcState->FSM_X[i], 0); break; case 1: T = _mm_extract_epi32(pSrcState->FSM_X[i], 1); break; case 2: T = _mm_extract_epi32(pSrcState->FSM_X[i], 2); break; case 3: T = _mm_extract_epi32(pSrcState->FSM_X[i], 3); break; } pDstState->FSM_R2 = T; i = 2; switch (NumBuffers) { case 0: T = _mm_extract_epi32(pSrcState->FSM_X[i], 0); break; case 1: T = _mm_extract_epi32(pSrcState->FSM_X[i], 1); break; case 2: T = _mm_extract_epi32(pSrcState->FSM_X[i], 2); break; case 3: T = _mm_extract_epi32(pSrcState->FSM_X[i], 3); break; } pDstState->FSM_R3 = T; } /*--------------------------------------------------------- * f8() * Initializations and Context size definitions *---------------------------------------------------------*/ size_t SNOW3G_KEY_SCHED_SIZE(void) { return sizeof(snow3g_key_schedule_t); } int SNOW3G_INIT_KEY_SCHED(const void *pKey, snow3g_key_schedule_t *pCtx) { #ifdef SAFE_PARAM if ((pKey == NULL) || (pCtx == NULL)) return -1; #endif const uint32_t *pKey32 = pKey; pCtx->k[3] = BSWAP32(pKey32[0]); pCtx->k[2] = BSWAP32(pKey32[1]); pCtx->k[1] = BSWAP32(pKey32[2]); pCtx->k[0] = BSWAP32(pKey32[3]); return 0; } /*--------------------------------------------------------- * @description * Snow3G F8 1 buffer: * Single buffer enc/dec with IV and precomputed key schedule *---------------------------------------------------------*/ void SNOW3G_F8_1_BUFFER(const snow3g_key_schedule_t *pHandle, const void *pIV, const void *pBufferIn, void *pBufferOut, const uint32_t lengthInBytes) { #ifdef SAFE_PARAM if ((pHandle == NULL) || (pIV == NULL) || (pBufferIn == NULL) || (pBufferOut == NULL) || (lengthInBytes == 0) || (lengthInBytes > SNOW3G_MAX_BYTELEN)) return; #endif snow3gKeyState1_t ctx; uint32_t KS4; /* 4 bytes of keystream */ /* Initialize the schedule from the IV */ snow3gStateInitialize_1(&ctx, pHandle, pIV); /* Clock FSM and LFSR once, ignore the keystream */ snow3g_keystream_1_4(&ctx, &KS4); f8_snow3g(&ctx, pBufferIn, pBufferOut, lengthInBytes); #ifdef SAFE_DATA CLEAR_VAR(&KS4, sizeof(KS4)); CLEAR_MEM(&ctx, sizeof(ctx)); CLEAR_SCRATCH_GPS(); CLEAR_SCRATCH_SIMD_REGS(); #endif /* SAFE_DATA */ } /*--------------------------------------------------------- * @description * Snow3G F8 bit 1 buffer: * Single buffer enc/dec with IV and precomputed key schedule *---------------------------------------------------------*/ void SNOW3G_F8_1_BUFFER_BIT(const snow3g_key_schedule_t *pHandle, const void *pIV, const void *pBufferIn, void *pBufferOut, const uint32_t lengthInBits, const uint32_t offsetInBits) { #ifdef SAFE_PARAM if ((pHandle == NULL) || (pIV == NULL) || (pBufferIn == NULL) || (pBufferOut == NULL) || (lengthInBits == 0)) return; #endif snow3gKeyState1_t ctx; uint32_t KS4; /* 4 bytes of keystream */ /* Initialize the schedule from the IV */ snow3gStateInitialize_1(&ctx, pHandle, pIV); /* Clock FSM and LFSR once, ignore the keystream */ snow3g_keystream_1_4(&ctx, &KS4); f8_snow3g_bit(&ctx, pBufferIn, pBufferOut, lengthInBits, offsetInBits); #ifdef SAFE_DATA CLEAR_VAR(&KS4, sizeof(KS4)); CLEAR_MEM(&ctx, sizeof(ctx)); CLEAR_SCRATCH_GPS(); CLEAR_SCRATCH_SIMD_REGS(); #endif /* SAFE_DATA */ } /*--------------------------------------------------------- * @description * Snow3G F8 2 buffer: * Two buffers enc/dec with the same key schedule. * The 3 IVs are independent and are passed as an array of pointers. * Each buffer and data length are separate. *---------------------------------------------------------*/ void SNOW3G_F8_2_BUFFER(const snow3g_key_schedule_t *pHandle, const void *pIV1, const void *pIV2, const void *pBufIn1, void *pBufOut1, const uint32_t lenInBytes1, const void *pBufIn2, void *pBufOut2, const uint32_t lenInBytes2) { #ifdef SAFE_PARAM if ((pHandle == NULL) || (pIV1 == NULL) || (pIV2 == NULL) || (pBufIn1 == NULL) || (pBufOut1 == NULL) || (pBufIn2 == NULL) || (pBufOut2 == NULL) || (lenInBytes1 == 0) || (lenInBytes1 > SNOW3G_MAX_BYTELEN) || (lenInBytes2 == 0) || (lenInBytes2 > SNOW3G_MAX_BYTELEN)) return; #endif snow3gKeyState1_t ctx1, ctx2; uint32_t KS4; /* 4 bytes of keystream */ /* Initialize the schedule from the IV */ snow3gStateInitialize_1(&ctx1, pHandle, pIV1); /* Clock FSM and LFSR once, ignore the keystream */ snow3g_keystream_1_4(&ctx1, &KS4); /* data processing for packet 1 */ f8_snow3g(&ctx1, pBufIn1, pBufOut1, lenInBytes1); /* Initialize the schedule from the IV */ snow3gStateInitialize_1(&ctx2, pHandle, pIV2); /* Clock FSM and LFSR once, ignore the keystream */ snow3g_keystream_1_4(&ctx2, &KS4); /* data processing for packet 2 */ f8_snow3g(&ctx2, pBufIn2, pBufOut2, lenInBytes2); #ifdef SAFE_DATA CLEAR_VAR(&KS4, sizeof(KS4)); CLEAR_MEM(&ctx1, sizeof(ctx1)); CLEAR_MEM(&ctx2, sizeof(ctx2)); CLEAR_SCRATCH_GPS(); CLEAR_SCRATCH_SIMD_REGS(); #endif /* SAFE_DATA */ } /*--------------------------------------------------------- * @description * Snow3G F8 4 buffer: * Four packets enc/dec with the same key schedule. * The 4 IVs are independent and are passed as an array of pointers. * Each buffer and data length are separate. *---------------------------------------------------------*/ void SNOW3G_F8_4_BUFFER(const snow3g_key_schedule_t *pHandle, const void *pIV1, const void *pIV2, const void *pIV3, const void *pIV4, const void *pBufferIn1, void *pBufferOut1, const uint32_t lengthInBytes1, const void *pBufferIn2, void *pBufferOut2, const uint32_t lengthInBytes2, const void *pBufferIn3, void *pBufferOut3, const uint32_t lengthInBytes3, const void *pBufferIn4, void *pBufferOut4, const uint32_t lengthInBytes4) { #ifdef SAFE_PARAM if ((pHandle == NULL) || (pIV1 == NULL) || (pIV2 == NULL) || (pIV3 == NULL) || (pIV4 == NULL) || (pBufferIn1 == NULL) || (pBufferOut1 == NULL) || (pBufferIn2 == NULL) || (pBufferOut2 == NULL) || (pBufferIn3 == NULL) || (pBufferOut3 == NULL) || (pBufferIn4 == NULL) || (pBufferOut4 == NULL) || (lengthInBytes1 == 0) || (lengthInBytes1 > SNOW3G_MAX_BYTELEN) || (lengthInBytes2 == 0) || (lengthInBytes2 > SNOW3G_MAX_BYTELEN) || (lengthInBytes3 == 0) || (lengthInBytes3 > SNOW3G_MAX_BYTELEN) || (lengthInBytes4 == 0) || (lengthInBytes4 > SNOW3G_MAX_BYTELEN)) return; #endif snow3gKeyState4_t ctx; __m128i H, L; /* 4 bytes of keystream */ uint32_t lenInBytes1 = lengthInBytes1; uint32_t lenInBytes2 = lengthInBytes2; uint32_t lenInBytes3 = lengthInBytes3; uint32_t lenInBytes4 = lengthInBytes4; uint32_t bytes1 = (lenInBytes1 < lenInBytes2 ? lenInBytes1 : lenInBytes2); /* number of bytes */ uint32_t bytes2 = (lenInBytes3 < lenInBytes4 ? lenInBytes3 : lenInBytes4); /* number of bytes */ /* min num of bytes */ uint32_t bytes = (bytes1 < bytes2) ? bytes1 : bytes2; uint32_t qwords = bytes / SNOW3G_8_BYTES; uint8_t *pBufOut1 = pBufferOut1; uint8_t *pBufOut2 = pBufferOut2; uint8_t *pBufOut3 = pBufferOut3; uint8_t *pBufOut4 = pBufferOut4; const uint8_t *pBufIn1 = pBufferIn1; const uint8_t *pBufIn2 = pBufferIn2; const uint8_t *pBufIn3 = pBufferIn3; const uint8_t *pBufIn4 = pBufferIn4; bytes = qwords * SNOW3G_8_BYTES; /* rounded down minimum length */ /* Initialize the schedule from the IV */ snow3gStateInitialize_4(&ctx, pHandle, pIV1, pIV2, pIV3, pIV4); /* Clock FSM and LFSR once, ignore the keystream */ snow3g_keystream_4_4(&ctx, &L); lenInBytes1 -= bytes; lenInBytes2 -= bytes; lenInBytes3 -= bytes; lenInBytes4 -= bytes; /* generates 4 bytes at a time on all streams */ while (qwords--) { snow3g_keystream_4_8(&ctx, &L, &H); pBufIn1 = xor_keystrm_rev(pBufOut1, pBufIn1, _mm_extract_epi64(L, 0)); pBufIn2 = xor_keystrm_rev(pBufOut2, pBufIn2, _mm_extract_epi64(L, 1)); pBufIn3 = xor_keystrm_rev(pBufOut3, pBufIn3, _mm_extract_epi64(H, 0)); pBufIn4 = xor_keystrm_rev(pBufOut4, pBufIn4, _mm_extract_epi64(H, 1)); pBufOut1 += SNOW3G_8_BYTES; pBufOut2 += SNOW3G_8_BYTES; pBufOut3 += SNOW3G_8_BYTES; pBufOut4 += SNOW3G_8_BYTES; } /* process the remaining of each buffer * - extract the LFSR and FSM structures * - Continue process 1 buffer */ if (lenInBytes1) { snow3gKeyState1_t ctx1; snow3gStateConvert_4(&ctx, &ctx1, 0); f8_snow3g(&ctx1, pBufIn1, pBufOut1, lenInBytes1); } if (lenInBytes2) { snow3gKeyState1_t ctx2; snow3gStateConvert_4(&ctx, &ctx2, 1); f8_snow3g(&ctx2, pBufIn2, pBufOut2, lenInBytes2); } if (lenInBytes3) { snow3gKeyState1_t ctx3; snow3gStateConvert_4(&ctx, &ctx3, 2); f8_snow3g(&ctx3, pBufIn3, pBufOut3, lenInBytes3); } if (lenInBytes4) { snow3gKeyState1_t ctx4; snow3gStateConvert_4(&ctx, &ctx4, 3); f8_snow3g(&ctx4, pBufIn4, pBufOut4, lenInBytes4); } #ifdef SAFE_DATA H = _mm_setzero_si128(); L = _mm_setzero_si128(); CLEAR_MEM(&ctx, sizeof(ctx)); CLEAR_SCRATCH_GPS(); CLEAR_SCRATCH_SIMD_REGS(); #endif /* SAFE_DATA */ } #ifdef AVX2 /*--------------------------------------------------------- * @description * Snow3G 8 buffer ks 8 multi: * Processes 8 packets 8 bytes at a time. * Uses individual key schedule for each buffer. *---------------------------------------------------------*/ static inline void snow3g_8_buffer_ks_8_multi(uint32_t bytes, const snow3g_key_schedule_t * const pKey[], const void * const IV[], const void * const pBufferIn[], void *pBufferOut[], const uint32_t *lengthInBytes) { uint32_t qwords = bytes / SNOW3G_8_BYTES; __m256i H, L; /* 8 bytes of keystream */ snow3gKeyState8_t ctx; int i; const uint8_t *tBufferIn[8]; uint8_t *tBufferOut[8]; uint32_t tLenInBytes[8]; bytes = qwords * SNOW3G_8_BYTES; /* rounded down minimum length */ for (i = 0; i < 8; i++) { tBufferIn[i] = pBufferIn[i]; tBufferOut[i] = pBufferOut[i]; tLenInBytes[i] = lengthInBytes[i]; } /* Initialize the schedule from the IV */ snow3gStateInitialize_8_multiKey(&ctx, pKey, IV); /* Clock FSM and LFSR once, ignore the keystream */ snow3g_keystream_8_4(&ctx, &L); for (i = 0; i < 8; i++) tLenInBytes[i] -= bytes; /* generates 8 sets at a time on all streams */ for (i = qwords; i != 0; i--) { int j; snow3g_keystream_8_8(&ctx, &L, &H); tBufferIn[0] = xor_keystrm_rev(tBufferOut[0], tBufferIn[0], _mm256_extract_epi64(L, 0)); tBufferIn[1] = xor_keystrm_rev(tBufferOut[1], tBufferIn[1], _mm256_extract_epi64(L, 1)); tBufferIn[2] = xor_keystrm_rev(tBufferOut[2], tBufferIn[2], _mm256_extract_epi64(H, 0)); tBufferIn[3] = xor_keystrm_rev(tBufferOut[3], tBufferIn[3], _mm256_extract_epi64(H, 1)); tBufferIn[4] = xor_keystrm_rev(tBufferOut[4], tBufferIn[4], _mm256_extract_epi64(L, 2)); tBufferIn[5] = xor_keystrm_rev(tBufferOut[5], tBufferIn[5], _mm256_extract_epi64(L, 3)); tBufferIn[6] = xor_keystrm_rev(tBufferOut[6], tBufferIn[6], _mm256_extract_epi64(H, 2)); tBufferIn[7] = xor_keystrm_rev(tBufferOut[7], tBufferIn[7], _mm256_extract_epi64(H, 3)); for (j = 0; j < 8; j++) tBufferOut[j] += SNOW3G_8_BYTES; } /* process the remaining of each buffer * - extract the LFSR and FSM structures * - Continue process 1 buffer */ if (tLenInBytes[0]) { snow3gKeyState1_t ctx1; snow3gStateConvert_8(&ctx, &ctx1, 0); f8_snow3g(&ctx1, tBufferIn[0], tBufferOut[0], tLenInBytes[0]); } if (tLenInBytes[1]) { snow3gKeyState1_t ctx2; snow3gStateConvert_8(&ctx, &ctx2, 1); f8_snow3g(&ctx2, tBufferIn[1], tBufferOut[1], tLenInBytes[1]); } if (tLenInBytes[2]) { snow3gKeyState1_t ctx3; snow3gStateConvert_8(&ctx, &ctx3, 2); f8_snow3g(&ctx3, tBufferIn[2], tBufferOut[2], tLenInBytes[2]); } if (tLenInBytes[3]) { snow3gKeyState1_t ctx4; snow3gStateConvert_8(&ctx, &ctx4, 3); f8_snow3g(&ctx4, tBufferIn[3], tBufferOut[3], tLenInBytes[3]); } if (tLenInBytes[4]) { snow3gKeyState1_t ctx5; snow3gStateConvert_8(&ctx, &ctx5, 4); f8_snow3g(&ctx5, tBufferIn[4], tBufferOut[4], tLenInBytes[4]); } if (tLenInBytes[5]) { snow3gKeyState1_t ctx6; snow3gStateConvert_8(&ctx, &ctx6, 5); f8_snow3g(&ctx6, tBufferIn[5], tBufferOut[5], tLenInBytes[5]); } if (tLenInBytes[6]) { snow3gKeyState1_t ctx7; snow3gStateConvert_8(&ctx, &ctx7, 6); f8_snow3g(&ctx7, tBufferIn[6], tBufferOut[6], tLenInBytes[6]); } if (tLenInBytes[7]) { snow3gKeyState1_t ctx8; snow3gStateConvert_8(&ctx, &ctx8, 7); f8_snow3g(&ctx8, tBufferIn[7], tBufferOut[7], tLenInBytes[7]); } #ifdef SAFE_DATA H = _mm256_setzero_si256(); L = _mm256_setzero_si256(); CLEAR_MEM(&ctx, sizeof(ctx)); #endif /* SAFE_DATA */ } /*--------------------------------------------------------- * @description * Snow3G 8 buffer ks 32 multi: * Processes 8 packets 32 bytes at a time. * Uses individual key schedule for each buffer. *---------------------------------------------------------*/ static inline void snow3g_8_buffer_ks_32_multi(uint32_t bytes, const snow3g_key_schedule_t * const pKey[], const void * const IV[], const void * const pBufferIn[], void *pBufferOut[], const uint32_t *lengthInBytes) { snow3gKeyState8_t ctx; uint32_t i; const uint8_t *tBufferIn[8]; uint8_t *tBufferOut[8]; uint32_t tLenInBytes[8]; for (i = 0; i < 8; i++) { tBufferIn[i] = pBufferIn[i]; tBufferOut[i] = pBufferOut[i]; tLenInBytes[i] = lengthInBytes[i]; } uint32_t blocks = bytes / 32; bytes = blocks * 32; /* rounded down minimum length */ /* Initialize the schedule from the IV */ snow3gStateInitialize_8_multiKey(&ctx, pKey, IV); /* Clock FSM and LFSR once, ignore the keystream */ __m256i ks[8]; snow3g_keystream_8_4(&ctx, ks); for (i = 0; i < 8; i++) tLenInBytes[i] -= bytes; __m256i in[8]; /* generates 8 sets at a time on all streams */ for (i = 0; i < blocks; i++) { int j; in[0] = _mm256_loadu_si256((const __m256i *)tBufferIn[0]); in[1] = _mm256_loadu_si256((const __m256i *)tBufferIn[1]); in[2] = _mm256_loadu_si256((const __m256i *)tBufferIn[2]); in[3] = _mm256_loadu_si256((const __m256i *)tBufferIn[3]); in[4] = _mm256_loadu_si256((const __m256i *)tBufferIn[4]); in[5] = _mm256_loadu_si256((const __m256i *)tBufferIn[5]); in[6] = _mm256_loadu_si256((const __m256i *)tBufferIn[6]); in[7] = _mm256_loadu_si256((const __m256i *)tBufferIn[7]); snow3g_keystream_8_32(&ctx, ks); _mm256_storeu_si256((__m256i *)tBufferOut[0], _mm256_xor_si256(in[0], ks[0])); _mm256_storeu_si256((__m256i *)tBufferOut[1], _mm256_xor_si256(in[1], ks[1])); _mm256_storeu_si256((__m256i *)tBufferOut[2], _mm256_xor_si256(in[2], ks[2])); _mm256_storeu_si256((__m256i *)tBufferOut[3], _mm256_xor_si256(in[3], ks[3])); _mm256_storeu_si256((__m256i *)tBufferOut[4], _mm256_xor_si256(in[4], ks[4])); _mm256_storeu_si256((__m256i *)tBufferOut[5], _mm256_xor_si256(in[5], ks[5])); _mm256_storeu_si256((__m256i *)tBufferOut[6], _mm256_xor_si256(in[6], ks[6])); _mm256_storeu_si256((__m256i *)tBufferOut[7], _mm256_xor_si256(in[7], ks[7])); for (j = 0; j < 8; j++) { tBufferIn[i] += 32; tBufferOut[i] += 32; } } /* process the remaining of each buffer * - extract the LFSR and FSM structures * - Continue process 1 buffer */ if (tLenInBytes[0]) { snow3gKeyState1_t ctx1; snow3gStateConvert_8(&ctx, &ctx1, 0); f8_snow3g(&ctx1, tBufferIn[0], tBufferOut[0], tLenInBytes[0]); } if (tLenInBytes[1]) { snow3gKeyState1_t ctx2; snow3gStateConvert_8(&ctx, &ctx2, 1); f8_snow3g(&ctx2, tBufferIn[1], tBufferOut[1], tLenInBytes[1]); } if (tLenInBytes[2]) { snow3gKeyState1_t ctx3; snow3gStateConvert_8(&ctx, &ctx3, 2); f8_snow3g(&ctx3, tBufferIn[2], tBufferOut[2], tLenInBytes[2]); } if (tLenInBytes[3]) { snow3gKeyState1_t ctx4; snow3gStateConvert_8(&ctx, &ctx4, 3); f8_snow3g(&ctx4, tBufferIn[3], tBufferOut[3], tLenInBytes[3]); } if (tLenInBytes[4]) { snow3gKeyState1_t ctx5; snow3gStateConvert_8(&ctx, &ctx5, 4); f8_snow3g(&ctx5, tBufferIn[4], tBufferOut[4], tLenInBytes[4]); } if (tLenInBytes[5]) { snow3gKeyState1_t ctx6; snow3gStateConvert_8(&ctx, &ctx6, 5); f8_snow3g(&ctx6, tBufferIn[5], tBufferOut[5], tLenInBytes[5]); } if (tLenInBytes[6]) { snow3gKeyState1_t ctx7; snow3gStateConvert_8(&ctx, &ctx7, 6); f8_snow3g(&ctx7, tBufferIn[6], tBufferOut[6], tLenInBytes[6]); } if (tLenInBytes[7]) { snow3gKeyState1_t ctx8; snow3gStateConvert_8(&ctx, &ctx8, 7); f8_snow3g(&ctx8, tBufferIn[7], tBufferOut[7], tLenInBytes[7]); } #ifdef SAFE_DATA CLEAR_MEM(&ctx, sizeof(ctx)); CLEAR_MEM(&ks, sizeof(ks)); CLEAR_MEM(&in, sizeof(in)); #endif /* SAFE_DATA */ } /*--------------------------------------------------------- * @description * Snow3G 8 buffer ks 8 multi: * Processes 8 packets 8 bytes at a time. * Uses same key schedule for each buffer. *---------------------------------------------------------*/ static inline void snow3g_8_buffer_ks_8(uint32_t bytes, const snow3g_key_schedule_t *pHandle, const void *pIV1, const void *pIV2, const void *pIV3, const void *pIV4, const void *pIV5, const void *pIV6, const void *pIV7, const void *pIV8, const void *pBufferIn1, void *pBufferOut1, const uint32_t lengthInBytes1, const void *pBufferIn2, void *pBufferOut2, const uint32_t lengthInBytes2, const void *pBufferIn3, void *pBufferOut3, const uint32_t lengthInBytes3, const void *pBufferIn4, void *pBufferOut4, const uint32_t lengthInBytes4, const void *pBufferIn5, void *pBufferOut5, const uint32_t lengthInBytes5, const void *pBufferIn6, void *pBufferOut6, const uint32_t lengthInBytes6, const void *pBufferIn7, void *pBufferOut7, const uint32_t lengthInBytes7, const void *pBufferIn8, void *pBufferOut8, const uint32_t lengthInBytes8) { uint32_t qwords = bytes / SNOW3G_8_BYTES; __m256i H, L; /* 8 bytes of keystream */ snow3gKeyState8_t ctx; int i; uint32_t lenInBytes1 = lengthInBytes1; uint32_t lenInBytes2 = lengthInBytes2; uint32_t lenInBytes3 = lengthInBytes3; uint32_t lenInBytes4 = lengthInBytes4; uint32_t lenInBytes5 = lengthInBytes5; uint32_t lenInBytes6 = lengthInBytes6; uint32_t lenInBytes7 = lengthInBytes7; uint32_t lenInBytes8 = lengthInBytes8; uint8_t *pBufOut1 = pBufferOut1; uint8_t *pBufOut2 = pBufferOut2; uint8_t *pBufOut3 = pBufferOut3; uint8_t *pBufOut4 = pBufferOut4; uint8_t *pBufOut5 = pBufferOut5; uint8_t *pBufOut6 = pBufferOut6; uint8_t *pBufOut7 = pBufferOut7; uint8_t *pBufOut8 = pBufferOut8; const uint8_t *pBufIn1 = pBufferIn1; const uint8_t *pBufIn2 = pBufferIn2; const uint8_t *pBufIn3 = pBufferIn3; const uint8_t *pBufIn4 = pBufferIn4; const uint8_t *pBufIn5 = pBufferIn5; const uint8_t *pBufIn6 = pBufferIn6; const uint8_t *pBufIn7 = pBufferIn7; const uint8_t *pBufIn8 = pBufferIn8; bytes = qwords * SNOW3G_8_BYTES; /* rounded down minimum length */ /* Initialize the schedule from the IV */ snow3gStateInitialize_8(&ctx, pHandle, pIV1, pIV2, pIV3, pIV4, pIV5, pIV6, pIV7, pIV8); /* Clock FSM and LFSR once, ignore the keystream */ snow3g_keystream_8_4(&ctx, &L); lenInBytes1 -= bytes; lenInBytes2 -= bytes; lenInBytes3 -= bytes; lenInBytes4 -= bytes; lenInBytes5 -= bytes; lenInBytes6 -= bytes; lenInBytes7 -= bytes; lenInBytes8 -= bytes; /* generates 8 sets at a time on all streams */ for (i = qwords; i != 0; i--) { snow3g_keystream_8_8(&ctx, &L, &H); pBufIn1 = xor_keystrm_rev(pBufOut1, pBufIn1, _mm256_extract_epi64(L, 0)); pBufIn2 = xor_keystrm_rev(pBufOut2, pBufIn2, _mm256_extract_epi64(L, 1)); pBufIn3 = xor_keystrm_rev(pBufOut3, pBufIn3, _mm256_extract_epi64(H, 0)); pBufIn4 = xor_keystrm_rev(pBufOut4, pBufIn4, _mm256_extract_epi64(H, 1)); pBufIn5 = xor_keystrm_rev(pBufOut5, pBufIn5, _mm256_extract_epi64(L, 2)); pBufIn6 = xor_keystrm_rev(pBufOut6, pBufIn6, _mm256_extract_epi64(L, 3)); pBufIn7 = xor_keystrm_rev(pBufOut7, pBufIn7, _mm256_extract_epi64(H, 2)); pBufIn8 = xor_keystrm_rev(pBufOut8, pBufIn8, _mm256_extract_epi64(H, 3)); pBufOut1 += SNOW3G_8_BYTES; pBufOut2 += SNOW3G_8_BYTES; pBufOut3 += SNOW3G_8_BYTES; pBufOut4 += SNOW3G_8_BYTES; pBufOut5 += SNOW3G_8_BYTES; pBufOut6 += SNOW3G_8_BYTES; pBufOut7 += SNOW3G_8_BYTES; pBufOut8 += SNOW3G_8_BYTES; } /* process the remaining of each buffer * - extract the LFSR and FSM structures * - Continue process 1 buffer */ if (lenInBytes1) { snow3gKeyState1_t ctx1; snow3gStateConvert_8(&ctx, &ctx1, 0); f8_snow3g(&ctx1, pBufIn1, pBufOut1, lenInBytes1); } if (lenInBytes2) { snow3gKeyState1_t ctx2; snow3gStateConvert_8(&ctx, &ctx2, 1); f8_snow3g(&ctx2, pBufIn2, pBufOut2, lenInBytes2); } if (lenInBytes3) { snow3gKeyState1_t ctx3; snow3gStateConvert_8(&ctx, &ctx3, 2); f8_snow3g(&ctx3, pBufIn3, pBufOut3, lenInBytes3); } if (lenInBytes4) { snow3gKeyState1_t ctx4; snow3gStateConvert_8(&ctx, &ctx4, 3); f8_snow3g(&ctx4, pBufIn4, pBufOut4, lenInBytes4); } if (lenInBytes5) { snow3gKeyState1_t ctx5; snow3gStateConvert_8(&ctx, &ctx5, 4); f8_snow3g(&ctx5, pBufIn5, pBufOut5, lenInBytes5); } if (lenInBytes6) { snow3gKeyState1_t ctx6; snow3gStateConvert_8(&ctx, &ctx6, 5); f8_snow3g(&ctx6, pBufIn6, pBufOut6, lenInBytes6); } if (lenInBytes7) { snow3gKeyState1_t ctx7; snow3gStateConvert_8(&ctx, &ctx7, 6); f8_snow3g(&ctx7, pBufIn7, pBufOut7, lenInBytes7); } if (lenInBytes8) { snow3gKeyState1_t ctx8; snow3gStateConvert_8(&ctx, &ctx8, 7); f8_snow3g(&ctx8, pBufIn8, pBufOut8, lenInBytes8); } #ifdef SAFE_DATA H = _mm256_setzero_si256(); L = _mm256_setzero_si256(); CLEAR_MEM(&ctx, sizeof(ctx)); #endif /* SAFE_DATA */ } /*--------------------------------------------------------- * @description * Snow3G 8 buffer ks 32 multi: * Processes 8 packets 32 bytes at a time. * Uses same key schedule for each buffer. *---------------------------------------------------------*/ static inline void snow3g_8_buffer_ks_32(uint32_t bytes, const snow3g_key_schedule_t *pKey, const void *pIV1, const void *pIV2, const void *pIV3, const void *pIV4, const void *pIV5, const void *pIV6, const void *pIV7, const void *pIV8, const void *pBufferIn1, void *pBufferOut1, const uint32_t lengthInBytes1, const void *pBufferIn2, void *pBufferOut2, const uint32_t lengthInBytes2, const void *pBufferIn3, void *pBufferOut3, const uint32_t lengthInBytes3, const void *pBufferIn4, void *pBufferOut4, const uint32_t lengthInBytes4, const void *pBufferIn5, void *pBufferOut5, const uint32_t lengthInBytes5, const void *pBufferIn6, void *pBufferOut6, const uint32_t lengthInBytes6, const void *pBufferIn7, void *pBufferOut7, const uint32_t lengthInBytes7, const void *pBufferIn8, void *pBufferOut8, const uint32_t lengthInBytes8) { snow3gKeyState8_t ctx; uint32_t i; uint32_t lenInBytes1 = lengthInBytes1; uint32_t lenInBytes2 = lengthInBytes2; uint32_t lenInBytes3 = lengthInBytes3; uint32_t lenInBytes4 = lengthInBytes4; uint32_t lenInBytes5 = lengthInBytes5; uint32_t lenInBytes6 = lengthInBytes6; uint32_t lenInBytes7 = lengthInBytes7; uint32_t lenInBytes8 = lengthInBytes8; uint8_t *pBufOut1 = pBufferOut1; uint8_t *pBufOut2 = pBufferOut2; uint8_t *pBufOut3 = pBufferOut3; uint8_t *pBufOut4 = pBufferOut4; uint8_t *pBufOut5 = pBufferOut5; uint8_t *pBufOut6 = pBufferOut6; uint8_t *pBufOut7 = pBufferOut7; uint8_t *pBufOut8 = pBufferOut8; const uint8_t *pBufIn1 = pBufferIn1; const uint8_t *pBufIn2 = pBufferIn2; const uint8_t *pBufIn3 = pBufferIn3; const uint8_t *pBufIn4 = pBufferIn4; const uint8_t *pBufIn5 = pBufferIn5; const uint8_t *pBufIn6 = pBufferIn6; const uint8_t *pBufIn7 = pBufferIn7; const uint8_t *pBufIn8 = pBufferIn8; uint32_t blocks = bytes / 32; bytes = blocks * 32; /* rounded down minimum length */ /* Initialize the schedule from the IV */ snow3gStateInitialize_8(&ctx, pKey, pIV1, pIV2, pIV3, pIV4, pIV5, pIV6, pIV7, pIV8); /* Clock FSM and LFSR once, ignore the keystream */ __m256i ks[8]; snow3g_keystream_8_4(&ctx, ks); lenInBytes1 -= bytes; lenInBytes2 -= bytes; lenInBytes3 -= bytes; lenInBytes4 -= bytes; lenInBytes5 -= bytes; lenInBytes6 -= bytes; lenInBytes7 -= bytes; lenInBytes8 -= bytes; __m256i in[8]; /* generates 8 sets at a time on all streams */ for (i = 0; i < blocks; i++) { in[0] = _mm256_loadu_si256((const __m256i *)pBufIn1); in[1] = _mm256_loadu_si256((const __m256i *)pBufIn2); in[2] = _mm256_loadu_si256((const __m256i *)pBufIn3); in[3] = _mm256_loadu_si256((const __m256i *)pBufIn4); in[4] = _mm256_loadu_si256((const __m256i *)pBufIn5); in[5] = _mm256_loadu_si256((const __m256i *)pBufIn6); in[6] = _mm256_loadu_si256((const __m256i *)pBufIn7); in[7] = _mm256_loadu_si256((const __m256i *)pBufIn8); snow3g_keystream_8_32(&ctx, ks); _mm256_storeu_si256((__m256i *)pBufOut1, _mm256_xor_si256(in[0], ks[0])); _mm256_storeu_si256((__m256i *)pBufOut2, _mm256_xor_si256(in[1], ks[1])); _mm256_storeu_si256((__m256i *)pBufOut3, _mm256_xor_si256(in[2], ks[2])); _mm256_storeu_si256((__m256i *)pBufOut4, _mm256_xor_si256(in[3], ks[3])); _mm256_storeu_si256((__m256i *)pBufOut5, _mm256_xor_si256(in[4], ks[4])); _mm256_storeu_si256((__m256i *)pBufOut6, _mm256_xor_si256(in[5], ks[5])); _mm256_storeu_si256((__m256i *)pBufOut7, _mm256_xor_si256(in[6], ks[6])); _mm256_storeu_si256((__m256i *)pBufOut8, _mm256_xor_si256(in[7], ks[7])); pBufIn1 += 32; pBufIn2 += 32; pBufIn3 += 32; pBufIn4 += 32; pBufIn5 += 32; pBufIn6 += 32; pBufIn7 += 32; pBufIn8 += 32; pBufOut1 += 32; pBufOut2 += 32; pBufOut3 += 32; pBufOut4 += 32; pBufOut5 += 32; pBufOut6 += 32; pBufOut7 += 32; pBufOut8 += 32; } /* process the remaining of each buffer * - extract the LFSR and FSM structures * - Continue process 1 buffer */ if (lenInBytes1) { snow3gKeyState1_t ctx1; snow3gStateConvert_8(&ctx, &ctx1, 0); f8_snow3g(&ctx1, pBufIn1, pBufOut1, lenInBytes1); } if (lenInBytes2) { snow3gKeyState1_t ctx2; snow3gStateConvert_8(&ctx, &ctx2, 1); f8_snow3g(&ctx2, pBufIn2, pBufOut2, lenInBytes2); } if (lenInBytes3) { snow3gKeyState1_t ctx3; snow3gStateConvert_8(&ctx, &ctx3, 2); f8_snow3g(&ctx3, pBufIn3, pBufOut3, lenInBytes3); } if (lenInBytes4) { snow3gKeyState1_t ctx4; snow3gStateConvert_8(&ctx, &ctx4, 3); f8_snow3g(&ctx4, pBufIn4, pBufOut4, lenInBytes4); } if (lenInBytes5) { snow3gKeyState1_t ctx5; snow3gStateConvert_8(&ctx, &ctx5, 4); f8_snow3g(&ctx5, pBufIn5, pBufOut5, lenInBytes5); } if (lenInBytes6) { snow3gKeyState1_t ctx6; snow3gStateConvert_8(&ctx, &ctx6, 5); f8_snow3g(&ctx6, pBufIn6, pBufOut6, lenInBytes6); } if (lenInBytes7) { snow3gKeyState1_t ctx7; snow3gStateConvert_8(&ctx, &ctx7, 6); f8_snow3g(&ctx7, pBufIn7, pBufOut7, lenInBytes7); } if (lenInBytes8) { snow3gKeyState1_t ctx8; snow3gStateConvert_8(&ctx, &ctx8, 7); f8_snow3g(&ctx8, pBufIn8, pBufOut8, lenInBytes8); } #ifdef SAFE_DATA CLEAR_MEM(&ctx, sizeof(ctx)); CLEAR_MEM(&ks, sizeof(ks)); CLEAR_MEM(&in, sizeof(in)); #endif /* SAFE_DATA */ } #endif /* AVX2 */ /*--------------------------------------------------------- * @description * Snow3G F8 8 buffer, multi-key: * Eight packets enc/dec with eight respective key schedules. * The 8 IVs are independent and are passed as an array of pointers. * Each buffer and data length are separate. *---------------------------------------------------------*/ void SNOW3G_F8_8_BUFFER_MULTIKEY(const snow3g_key_schedule_t * const pKey[], const void * const IV[], const void * const BufferIn[], void *BufferOut[], const uint32_t lengthInBytes[]) { int i; #ifdef SAFE_PARAM if ((pKey == NULL) || (IV == NULL) || (BufferIn == NULL) || (BufferOut == NULL) || (lengthInBytes == NULL)) return; for (i = 0; i < 8; i++) if ((pKey[i] == NULL) || (IV[i] == NULL) || (BufferIn[i] == NULL) || (BufferOut[i] == NULL) || (lengthInBytes[i] == 0) || (lengthInBytes[i] > SNOW3G_MAX_BYTELEN)) return; #endif #ifndef AVX2 /* basic C workaround for lack of non AVX2 implementation */ for (i = 0; i < 8; i++) SNOW3G_F8_1_BUFFER(pKey[i], IV[i], BufferIn[i], BufferOut[i], lengthInBytes[i]); #else uint32_t bytes = lengthInBytes[0]; /* find min byte lenght */ for (i = 1; i < 8; i++) if (lengthInBytes[i] < bytes) bytes = lengthInBytes[i]; if (bytes % 32) { snow3g_8_buffer_ks_8_multi(bytes, pKey, IV, BufferIn, BufferOut, lengthInBytes); } else { snow3g_8_buffer_ks_32_multi(bytes, pKey, IV, BufferIn, BufferOut, lengthInBytes); } #ifdef SAFE_DATA CLEAR_SCRATCH_GPS(); CLEAR_SCRATCH_SIMD_REGS(); #endif #endif /* AVX2 */ } /*--------------------------------------------------------- * @description * Snow3G F8 8 buffer: * Eight packets enc/dec with the same key schedule. * The 8 IVs are independent and are passed as an array of pointers. * Each buffer and data length are separate. * Uses AVX instructions. *---------------------------------------------------------*/ void SNOW3G_F8_8_BUFFER(const snow3g_key_schedule_t *pHandle, const void *pIV1, const void *pIV2, const void *pIV3, const void *pIV4, const void *pIV5, const void *pIV6, const void *pIV7, const void *pIV8, const void *pBufIn1, void *pBufOut1, const uint32_t lenInBytes1, const void *pBufIn2, void *pBufOut2, const uint32_t lenInBytes2, const void *pBufIn3, void *pBufOut3, const uint32_t lenInBytes3, const void *pBufIn4, void *pBufOut4, const uint32_t lenInBytes4, const void *pBufIn5, void *pBufOut5, const uint32_t lenInBytes5, const void *pBufIn6, void *pBufOut6, const uint32_t lenInBytes6, const void *pBufIn7, void *pBufOut7, const uint32_t lenInBytes7, const void *pBufIn8, void *pBufOut8, const uint32_t lenInBytes8) { #ifdef SAFE_PARAM if ((pHandle == NULL) || (pIV1 == NULL) || (pIV2 == NULL) || (pIV3 == NULL) || (pIV4 == NULL) || (pIV5 == NULL) || (pIV6 == NULL) || (pIV7 == NULL) || (pIV8 == NULL) || (pBufIn1 == NULL) || (pBufOut1 == NULL) || (pBufIn2 == NULL) || (pBufOut2 == NULL) || (pBufIn3 == NULL) || (pBufOut3 == NULL) || (pBufIn4 == NULL) || (pBufOut4 == NULL) || (pBufIn5 == NULL) || (pBufOut5 == NULL) || (pBufIn6 == NULL) || (pBufOut6 == NULL) || (pBufIn7 == NULL) || (pBufOut7 == NULL) || (pBufIn8 == NULL) || (pBufOut8 == NULL) || (lenInBytes1 == 0) || (lenInBytes1 > SNOW3G_MAX_BYTELEN) || (lenInBytes2 == 0) || (lenInBytes2 > SNOW3G_MAX_BYTELEN) || (lenInBytes3 == 0) || (lenInBytes3 > SNOW3G_MAX_BYTELEN) || (lenInBytes4 == 0) || (lenInBytes4 > SNOW3G_MAX_BYTELEN) || (lenInBytes5 == 0) || (lenInBytes5 > SNOW3G_MAX_BYTELEN) || (lenInBytes6 == 0) || (lenInBytes6 > SNOW3G_MAX_BYTELEN) || (lenInBytes7 == 0) || (lenInBytes7 > SNOW3G_MAX_BYTELEN) || (lenInBytes8 == 0) || (lenInBytes8 > SNOW3G_MAX_BYTELEN)) return; #endif #ifdef AVX2 uint32_t bytes1 = (lenInBytes1 < lenInBytes2 ? lenInBytes1 : lenInBytes2); /* number of bytes */ uint32_t bytes2 = (lenInBytes3 < lenInBytes4 ? lenInBytes3 : lenInBytes4); /* number of bytes */ uint32_t bytes3 = (lenInBytes5 < lenInBytes6 ? lenInBytes5 : lenInBytes6); /* number of bytes */ uint32_t bytes4 = (lenInBytes7 < lenInBytes8 ? lenInBytes7 : lenInBytes8); /* number of bytes */ uint32_t bytesq1 = (bytes1 < bytes2) ? bytes1 : bytes2; /* min number of bytes */ uint32_t bytesq2 = (bytes3 < bytes4) ? bytes3 : bytes4; uint32_t bytes = (bytesq1 < bytesq2) ? bytesq1 : bytesq2; if (bytes % 32) { snow3g_8_buffer_ks_8( bytes, pHandle, pIV1, pIV2, pIV3, pIV4, pIV5, pIV6, pIV7, pIV8, pBufIn1, pBufOut1, lenInBytes1, pBufIn2, pBufOut2, lenInBytes2, pBufIn3, pBufOut3, lenInBytes3, pBufIn4, pBufOut4, lenInBytes4, pBufIn5, pBufOut5, lenInBytes5, pBufIn6, pBufOut6, lenInBytes6, pBufIn7, pBufOut7, lenInBytes7, pBufIn8, pBufOut8, lenInBytes8); } else { snow3g_8_buffer_ks_32( bytes, pHandle, pIV1, pIV2, pIV3, pIV4, pIV5, pIV6, pIV7, pIV8, pBufIn1, pBufOut1, lenInBytes1, pBufIn2, pBufOut2, lenInBytes2, pBufIn3, pBufOut3, lenInBytes3, pBufIn4, pBufOut4, lenInBytes4, pBufIn5, pBufOut5, lenInBytes5, pBufIn6, pBufOut6, lenInBytes6, pBufIn7, pBufOut7, lenInBytes7, pBufIn8, pBufOut8, lenInBytes8); } #ifdef SAFE_DATA CLEAR_SCRATCH_GPS(); CLEAR_SCRATCH_SIMD_REGS(); #endif #else /* ~AVX2 */ SNOW3G_F8_2_BUFFER(pHandle, pIV1, pIV2, pBufIn1, pBufOut1, lenInBytes1, pBufIn2, pBufOut2, lenInBytes2); SNOW3G_F8_2_BUFFER(pHandle, pIV3, pIV4, pBufIn3, pBufOut3, lenInBytes3, pBufIn4, pBufOut4, lenInBytes4); SNOW3G_F8_2_BUFFER(pHandle, pIV5, pIV6, pBufIn5, pBufOut5, lenInBytes5, pBufIn6, pBufOut6, lenInBytes6); SNOW3G_F8_2_BUFFER(pHandle, pIV7, pIV8, pBufIn7, pBufOut7, lenInBytes7, pBufIn8, pBufOut8, lenInBytes8); #endif /* AVX */ } /****************************************************************************** * @description * Snow3G F8 multi packet: * Performs F8 enc/dec on [n] packets. The operation is performed in-place. * The input IV's are passed in Little Endian format. * The KeySchedule is in Little Endian format. ******************************************************************************/ void SNOW3G_F8_N_BUFFER(const snow3g_key_schedule_t *pCtx, const void * const IV[], const void * const pBufferIn[], void *pBufferOut[], const uint32_t bufLenInBytes[], const uint32_t packetCount) { #ifdef SAFE_PARAM uint32_t i; if ((pCtx == NULL) || (IV == NULL) || (pBufferIn == NULL) || (pBufferOut == NULL) || (bufLenInBytes == NULL)) return; for (i = 0; i < packetCount; i++) if ((IV[i] == NULL) || (pBufferIn[i] == NULL) || (pBufferOut[i] == NULL) || (bufLenInBytes[i] == 0) || (bufLenInBytes[i] > SNOW3G_MAX_BYTELEN)) return; #endif if (packetCount > 16) { pBufferOut[0] = NULL; printf("packetCount too high (%d)\n", packetCount); return; } uint32_t packet_index, inner_index, pktCnt = packetCount; int sortNeeded = 0, tempLen = 0; uint8_t *srctempbuff; uint8_t *dsttempbuff; uint8_t *ivtempbuff; uint8_t *pSrcBuf[NUM_PACKETS_16] = {NULL}; uint8_t *pDstBuf[NUM_PACKETS_16] = {NULL}; uint8_t *pIV[NUM_PACKETS_16] = {NULL}; uint32_t lensBuf[NUM_PACKETS_16] = {0}; memcpy((void *)lensBuf, bufLenInBytes, packetCount * sizeof(uint32_t)); memcpy((void *)pSrcBuf, pBufferIn, packetCount * sizeof(void *)); memcpy((void *)pDstBuf, pBufferOut, packetCount * sizeof(void *)); memcpy((void *)pIV, IV, packetCount * sizeof(void *)); packet_index = packetCount; while (packet_index--) { /* check if all packets are sorted by decreasing length */ if (packet_index > 0 && lensBuf[packet_index - 1] < lensBuf[packet_index]) { /* this packet array is not correctly sorted */ sortNeeded = 1; } } if (sortNeeded) { /* sort packets in decreasing buffer size from [0] to [n]th packet, ** where buffer[0] will contain longest buffer and buffer[n] will contain the shortest buffer. 4 arrays are swapped : - pointers to input buffers - pointers to output buffers - pointers to input IV's - input buffer lengths */ packet_index = packetCount; while (packet_index--) { inner_index = packet_index; while (inner_index--) { if (lensBuf[packet_index] > lensBuf[inner_index]) { /* swap buffers to arrange in descending order from [0]. */ srctempbuff = pSrcBuf[packet_index]; dsttempbuff = pDstBuf[packet_index]; ivtempbuff = pIV[packet_index]; tempLen = lensBuf[packet_index]; pSrcBuf[packet_index] = pSrcBuf[inner_index]; pDstBuf[packet_index] = pDstBuf[inner_index]; pIV[packet_index] = pIV[inner_index]; lensBuf[packet_index] = lensBuf[inner_index]; pSrcBuf[inner_index] = srctempbuff; pDstBuf[inner_index] = dsttempbuff; pIV[inner_index] = ivtempbuff; lensBuf[inner_index] = tempLen; } } /* for inner packet index (inner bubble-sort) */ } /* for outer packet index (outer bubble-sort) */ } /* if sortNeeded */ packet_index = 0; /* process 8 buffers at-a-time */ #ifdef AVX2 while (pktCnt >= 8) { pktCnt -= 8; SNOW3G_F8_8_BUFFER(pCtx, pIV[packet_index], pIV[packet_index + 1], pIV[packet_index + 2], pIV[packet_index + 3], pIV[packet_index + 4], pIV[packet_index + 5], pIV[packet_index + 6], pIV[packet_index + 7], pSrcBuf[packet_index], pDstBuf[packet_index], lensBuf[packet_index], pSrcBuf[packet_index + 1], pDstBuf[packet_index + 1], lensBuf[packet_index + 1], pSrcBuf[packet_index + 2], pDstBuf[packet_index + 2], lensBuf[packet_index + 2], pSrcBuf[packet_index + 3], pDstBuf[packet_index + 3], lensBuf[packet_index + 3], pSrcBuf[packet_index + 4], pDstBuf[packet_index + 4], lensBuf[packet_index + 4], pSrcBuf[packet_index + 5], pDstBuf[packet_index + 5], lensBuf[packet_index + 5], pSrcBuf[packet_index + 6], pDstBuf[packet_index + 6], lensBuf[packet_index + 6], pSrcBuf[packet_index + 7], pDstBuf[packet_index + 7], lensBuf[packet_index + 7]); packet_index += 8; } #endif /* process 4 buffers at-a-time */ while (pktCnt >= 4) { pktCnt -= 4; SNOW3G_F8_4_BUFFER(pCtx, pIV[packet_index + 0], pIV[packet_index + 1], pIV[packet_index + 2], pIV[packet_index + 3], pSrcBuf[packet_index + 0], pDstBuf[packet_index + 0], lensBuf[packet_index + 0], pSrcBuf[packet_index + 1], pDstBuf[packet_index + 1], lensBuf[packet_index + 1], pSrcBuf[packet_index + 2], pDstBuf[packet_index + 2], lensBuf[packet_index + 2], pSrcBuf[packet_index + 3], pDstBuf[packet_index + 3], lensBuf[packet_index + 3]); packet_index += 4; } /* process 2 packets at-a-time */ while (pktCnt >= 2) { pktCnt -= 2; SNOW3G_F8_2_BUFFER(pCtx, pIV[packet_index + 0], pIV[packet_index + 1], pSrcBuf[packet_index + 0], pDstBuf[packet_index + 0], lensBuf[packet_index + 0], pSrcBuf[packet_index + 1], pDstBuf[packet_index + 1], lensBuf[packet_index + 1]); packet_index += 2; } /* remaining packets are processed 1 at a time */ while (pktCnt--) { SNOW3G_F8_1_BUFFER(pCtx, pIV[packet_index + 0], pSrcBuf[packet_index + 0], pDstBuf[packet_index + 0], lensBuf[packet_index + 0]); packet_index++; } } void SNOW3G_F8_N_BUFFER_MULTIKEY(const snow3g_key_schedule_t * const pCtx[], const void * const IV[], const void * const pBufferIn[], void *pBufferOut[], const uint32_t bufLenInBytes[], const uint32_t packetCount) { #ifdef SAFE_PARAM uint32_t i; if ((pCtx == NULL) || (IV == NULL) || (pBufferIn == NULL) || (pBufferOut == NULL) || (bufLenInBytes == NULL)) return; for (i = 0; i < packetCount; i++) if ((pCtx[i] == NULL) || (IV[i] == NULL) || (pBufferIn[i] == NULL) || (pBufferOut[i] == NULL) || (bufLenInBytes[i] == 0) || (bufLenInBytes[i] > SNOW3G_MAX_BYTELEN)) return; #endif if (packetCount > 16) { pBufferOut[0] = NULL; printf("packetCount too high (%d)\n", packetCount); return; } uint32_t packet_index, inner_index, pktCnt = packetCount; int sortNeeded = 0, tempLen = 0; uint8_t *srctempbuff; uint8_t *dsttempbuff; uint8_t *ivtempbuff; snow3g_key_schedule_t *pCtxBuf[NUM_PACKETS_16] = {NULL}; uint8_t *pSrcBuf[NUM_PACKETS_16] = {NULL}; uint8_t *pDstBuf[NUM_PACKETS_16] = {NULL}; uint8_t *pIV[NUM_PACKETS_16] = {NULL}; uint32_t lensBuf[NUM_PACKETS_16] = {0}; snow3g_key_schedule_t *tempCtx; memcpy((void *)pCtxBuf, pCtx, packetCount * sizeof(void *)); memcpy((void *)lensBuf, bufLenInBytes, packetCount * sizeof(uint32_t)); memcpy((void *)pSrcBuf, pBufferIn, packetCount * sizeof(void *)); memcpy((void *)pDstBuf, pBufferOut, packetCount * sizeof(void *)); memcpy((void *)pIV, IV, packetCount * sizeof(void *)); packet_index = packetCount; while (packet_index--) { /* check if all packets are sorted by decreasing length */ if (packet_index > 0 && lensBuf[packet_index - 1] < lensBuf[packet_index]) { /* this packet array is not correctly sorted */ sortNeeded = 1; } } if (sortNeeded) { /* sort packets in decreasing buffer size from [0] to [n]th packet, where buffer[0] will contain longest buffer and buffer[n] will contain the shortest buffer. 4 arrays are swapped : - pointers to input buffers - pointers to output buffers - pointers to input IV's - input buffer lengths */ packet_index = packetCount; while (packet_index--) { inner_index = packet_index; while (inner_index--) { if (lensBuf[packet_index] > lensBuf[inner_index]) { /* swap buffers to arrange in descending order from [0]. */ srctempbuff = pSrcBuf[packet_index]; dsttempbuff = pDstBuf[packet_index]; ivtempbuff = pIV[packet_index]; tempLen = lensBuf[packet_index]; tempCtx = pCtxBuf[packet_index]; pSrcBuf[packet_index] = pSrcBuf[inner_index]; pDstBuf[packet_index] = pDstBuf[inner_index]; pIV[packet_index] = pIV[inner_index]; lensBuf[packet_index] = lensBuf[inner_index]; pCtxBuf[packet_index] = pCtxBuf[inner_index]; pSrcBuf[inner_index] = srctempbuff; pDstBuf[inner_index] = dsttempbuff; pIV[inner_index] = ivtempbuff; lensBuf[inner_index] = tempLen; pCtxBuf[inner_index] = tempCtx; } } /* for inner packet index (inner bubble-sort) */ } /* for outer packet index (outer bubble-sort) */ } /* if sortNeeded */ packet_index = 0; /* process 8 buffers at-a-time */ #ifdef AVX2 while (pktCnt >= 8) { pktCnt -= 8; SNOW3G_F8_8_BUFFER_MULTIKEY( (const snow3g_key_schedule_t * const *) &pCtxBuf[packet_index], (const void * const *)&pIV[packet_index], (const void * const *)&pSrcBuf[packet_index], (void **)&pDstBuf[packet_index], &lensBuf[packet_index]); packet_index += 8; } #endif /* TODO process 4 buffers at-a-time */ /* TODO process 2 packets at-a-time */ /* remaining packets are processed 1 at a time */ while (pktCnt--) { SNOW3G_F8_1_BUFFER(pCtxBuf[packet_index + 0], pIV[packet_index + 0], pSrcBuf[packet_index + 0], pDstBuf[packet_index + 0], lensBuf[packet_index + 0]); packet_index++; } } /*--------------------------------------------------------- * @description * Snow3G F9 1 buffer * Single buffer digest with IV and precomputed key schedule *---------------------------------------------------------*/ void SNOW3G_F9_1_BUFFER(const snow3g_key_schedule_t *pHandle, const void *pIV, const void *pBufferIn, const uint64_t lengthInBits, void *pDigest) { #ifdef SAFE_PARAM if ((pHandle == NULL) || (pIV == NULL) || (pBufferIn == NULL) || (pDigest == NULL) || (lengthInBits == 0) || (lengthInBits > SNOW3G_MAX_BITLEN)) return; #endif snow3gKeyState1_t ctx; uint32_t z[5]; uint64_t lengthInQwords, E, V, P; uint64_t i, rem_bits; const uint64_t *inputBuffer; inputBuffer = (const uint64_t *)pBufferIn; /* Initialize the snow3g key schedule */ snow3gStateInitialize_1(&ctx, pHandle, pIV); /*Generate 5 keystream words*/ snow3g_f9_keystream_words(&ctx, &z[0]); P = ((uint64_t)z[0] << 32) | ((uint64_t)z[1]); lengthInQwords = lengthInBits / 64; E = 0; /* all blocks except the last one */ for (i = 0; i < lengthInQwords; i++) { V = BSWAP64(inputBuffer[i]); E = multiply_and_reduce64(E ^ V, P); } /* last bits of last block if any left */ rem_bits = lengthInBits % 64; if (rem_bits) { /* last bytes, do not go past end of buffer */ memcpy(&V, &inputBuffer[i], (rem_bits + 7) / 8); V = BSWAP64(V); V &= (((uint64_t)-1) << (64 - rem_bits)); /* mask extra bits */ E = multiply_and_reduce64(E ^ V, P); } /* Multiply by Q */ E = multiply_and_reduce64(E ^ lengthInBits, (((uint64_t)z[2] << 32) | ((uint64_t)z[3]))); /* Final MAC */ *(uint32_t *)pDigest = (uint32_t)BSWAP64(E ^ ((uint64_t)z[4] << 32)); #ifdef SAFE_DATA CLEAR_VAR(&E, sizeof(E)); CLEAR_VAR(&V, sizeof(V)); CLEAR_VAR(&P, sizeof(P)); CLEAR_MEM(&z, sizeof(z)); CLEAR_MEM(&ctx, sizeof(ctx)); CLEAR_SCRATCH_GPS(); CLEAR_SCRATCH_SIMD_REGS(); #endif /* SAFE_DATA */ } #endif /* SNOW3G_COMMON_H */