1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
|
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
#ifdef FREEBL_NO_DEPEND
#include "stubs.h"
#endif
#include "gcm.h"
#include "secerr.h"
#include <wmmintrin.h> /* clmul */
#define WRITE64(x, bytes) \
(bytes)[0] = (x) >> 56; \
(bytes)[1] = (x) >> 48; \
(bytes)[2] = (x) >> 40; \
(bytes)[3] = (x) >> 32; \
(bytes)[4] = (x) >> 24; \
(bytes)[5] = (x) >> 16; \
(bytes)[6] = (x) >> 8; \
(bytes)[7] = (x);
SECStatus
gcm_HashWrite_hw(gcmHashContext *ghash, unsigned char *outbuf)
{
uint64_t tmp_out[2];
_mm_storeu_si128((__m128i *)tmp_out, ghash->x);
/* maxout must be larger than 16 byte (checked by the caller). */
WRITE64(tmp_out[0], outbuf + 8);
WRITE64(tmp_out[1], outbuf);
return SECSuccess;
}
SECStatus
gcm_HashMult_hw(gcmHashContext *ghash, const unsigned char *buf,
unsigned int count)
{
size_t i;
pre_align __m128i z_high post_align;
pre_align __m128i z_low post_align;
pre_align __m128i C post_align;
pre_align __m128i D post_align;
pre_align __m128i E post_align;
pre_align __m128i F post_align;
pre_align __m128i bin post_align;
pre_align __m128i Ci post_align;
pre_align __m128i tmp post_align;
for (i = 0; i < count; i++, buf += 16) {
bin = _mm_set_epi16(((uint16_t)buf[0] << 8) | buf[1],
((uint16_t)buf[2] << 8) | buf[3],
((uint16_t)buf[4] << 8) | buf[5],
((uint16_t)buf[6] << 8) | buf[7],
((uint16_t)buf[8] << 8) | buf[9],
((uint16_t)buf[10] << 8) | buf[11],
((uint16_t)buf[12] << 8) | buf[13],
((uint16_t)buf[14] << 8) | buf[15]);
Ci = _mm_xor_si128(bin, ghash->x);
/* Do binary mult ghash->X = Ci * ghash->H. */
C = _mm_clmulepi64_si128(Ci, ghash->h, 0x00);
D = _mm_clmulepi64_si128(Ci, ghash->h, 0x11);
E = _mm_clmulepi64_si128(Ci, ghash->h, 0x01);
F = _mm_clmulepi64_si128(Ci, ghash->h, 0x10);
tmp = _mm_xor_si128(E, F);
z_high = _mm_xor_si128(tmp, _mm_slli_si128(D, 8));
z_high = _mm_unpackhi_epi64(z_high, D);
z_low = _mm_xor_si128(_mm_slli_si128(tmp, 8), C);
z_low = _mm_unpackhi_epi64(_mm_slli_si128(C, 8), z_low);
/* Shift one to the left (multiply by x) as gcm spec is stupid. */
C = _mm_slli_si128(z_low, 8);
E = _mm_srli_epi64(C, 63);
D = _mm_slli_si128(z_high, 8);
F = _mm_srli_epi64(D, 63);
/* Carry over */
C = _mm_srli_si128(z_low, 8);
D = _mm_srli_epi64(C, 63);
z_low = _mm_or_si128(_mm_slli_epi64(z_low, 1), E);
z_high = _mm_or_si128(_mm_or_si128(_mm_slli_epi64(z_high, 1), F), D);
/* Reduce */
C = _mm_slli_si128(z_low, 8);
/* D = z_low << 127 */
D = _mm_slli_epi64(C, 63);
/* E = z_low << 126 */
E = _mm_slli_epi64(C, 62);
/* F = z_low << 121 */
F = _mm_slli_epi64(C, 57);
/* z_low ^= (z_low << 127) ^ (z_low << 126) ^ (z_low << 121); */
z_low = _mm_xor_si128(_mm_xor_si128(_mm_xor_si128(z_low, D), E), F);
C = _mm_srli_si128(z_low, 8);
/* D = z_low >> 1 */
D = _mm_slli_epi64(C, 63);
D = _mm_or_si128(_mm_srli_epi64(z_low, 1), D);
/* E = z_low >> 2 */
E = _mm_slli_epi64(C, 62);
E = _mm_or_si128(_mm_srli_epi64(z_low, 2), E);
/* F = z_low >> 7 */
F = _mm_slli_epi64(C, 57);
F = _mm_or_si128(_mm_srli_epi64(z_low, 7), F);
/* ghash->x ^= z_low ^ (z_low >> 1) ^ (z_low >> 2) ^ (z_low >> 7); */
ghash->x = _mm_xor_si128(_mm_xor_si128(
_mm_xor_si128(_mm_xor_si128(z_high, z_low), D), E),
F);
}
return SECSuccess;
}
SECStatus
gcm_HashInit_hw(gcmHashContext *ghash)
{
ghash->ghash_mul = gcm_HashMult_hw;
ghash->x = _mm_setzero_si128();
/* MSVC requires __m64 to load epi64. */
ghash->h = _mm_set_epi32(ghash->h_high >> 32, (uint32_t)ghash->h_high,
ghash->h_low >> 32, (uint32_t)ghash->h_low);
ghash->hw = PR_TRUE;
return SECSuccess;
}
SECStatus
gcm_HashZeroX_hw(gcmHashContext *ghash)
{
ghash->x = _mm_setzero_si128();
return SECSuccess;
}
|
