diff options
Diffstat (limited to '')
-rw-r--r-- | src/crypto/isa-l/isa-l_crypto/aes/aarch64/gcm_common.S | 430 |
1 files changed, 430 insertions, 0 deletions
diff --git a/src/crypto/isa-l/isa-l_crypto/aes/aarch64/gcm_common.S b/src/crypto/isa-l/isa-l_crypto/aes/aarch64/gcm_common.S new file mode 100644 index 000000000..042f6cf19 --- /dev/null +++ b/src/crypto/isa-l/isa-l_crypto/aes/aarch64/gcm_common.S @@ -0,0 +1,430 @@ +/********************************************************************** + Copyright(c) 2021 Arm Corporation All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Arm Corporation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +**********************************************************************/ + .arch armv8-a+crypto + .text +#define HASHKEY_TOTAL_NUM (24) +#define HASHKEY_BASE_OFF (15*16) +#define HASHKEY_OFF(n) ((15*16)+n*32) +#define HASHKEY_EXT_OFF(n) ((15*16)+n*32+16) +#ifndef KEY_LEN +#define KEY_LEN 128 +#endif +#ifndef BLOCKS +#define BLOCKS 24 +#endif +#define FN_NAME(fn,mode,post) aes_gcm_##fn##_##mode####post##aes +#define START_FUNC(fn,mode,post) .global FN_NAME(fn,mode,post); \ + .type FN_NAME(fn,mode,post), %function; \ + FN_NAME(fn,mode,post): +#define END_FUNC(fn,mode,post) .size FN_NAME(fn,mode,post), .-FN_NAME(fn,mode,post) + +#define AAD_LEN_OFF 16 +#define IN_LENGTH_OFF 24 +#define PARTIAL_BLOCK_ENC_KEY_OFF 32 +#define PARTIAL_BLOCK_LENGTH_OFF 80 +#define CTR_OFF 64 +#define ORIG_IV_OFF 48 +/* + [low,middle,tmp0,high] +=dat0 * [hashkey0,hashkey0_ext] + ifnb dat1 + dat1=rbit(*dat_adr) + [hashkey0,hashkey0_ext] = *hashkey_adr + dat_adr+=16 + hashkey_adr+=32 +*/ + +.macro ghash_mult_round aadhash:req,dat_adr:req,hashkey_adr:req, \ + hashkey0:req,hashkey0_ext:req,high:req,low:req,middle:req, \ + tmp0:req,tmp1:req,next_dat:req,left_count:req + + ldr q\next_dat,[\dat_adr],16 + pmull v\tmp0\().1q,v\aadhash\().1d,v\hashkey0_ext\().1d + pmull2 v\tmp1\().1q,v\aadhash\().2d,v\hashkey0_ext\().2d + .if \left_count > 1 + ldr q\hashkey0_ext,[\hashkey_adr,16] + .endif + eor v\middle\().16b,v\middle\().16b,v\tmp0\().16b + pmull2 v\tmp0\().1q,v\aadhash\().2d,v\hashkey0\().2d + eor v\middle\().16b,v\middle\().16b,v\tmp1\().16b + pmull v\tmp1\().1q,v\aadhash\().1d,v\hashkey0\().1d + .if \left_count > 1 + ldr q\hashkey0,[\hashkey_adr],32 + .endif + eor v\high\().16b,v\high\().16b,v\tmp0\().16b + eor v\low\().16b,v\low\().16b,v\tmp1\().16b + rbit v\aadhash\().16b, v\next_dat\().16b +.endm + +.macro ghash_mult_init_round aadhash:req,dat_adr:req,hashkey_adr:req, \ + hashkey0:req,hashkey0_ext:req, \ + high:req,low:req,middle:req,tmp0:req,next_dat:req,left_count:req + ldp q\hashkey0,q\hashkey0_ext,[\hashkey_adr],32 + ldr q\next_dat,[\dat_adr],16 + pmull v\middle\().1q,v\aadhash\().1d,v\hashkey0_ext\().1d + pmull2 v\tmp0\().1q,v\aadhash\().2d,v\hashkey0_ext\().2d + .if \left_count > 1 + ldr q\hashkey0_ext,[\hashkey_adr,16] + .endif + pmull2 v\high\().1q,v\aadhash\().2d,v\hashkey0\().2d + eor v\middle\().16b,v\middle\().16b,v\tmp0\().16b + + pmull v\low\().1q,v\aadhash\().1d,v\hashkey0\().1d + .if \left_count > 1 + ldr q\hashkey0,[\hashkey_adr],32 + .endif + rbit v\aadhash\().16b, v\next_dat\().16b +.endm + +/* aadhash=reduction(low,middle,high)+dat0 */ +.macro ghash_mult_final_round aadhash:req, \ + high:req,low:req,middle:req,tmp0:req, \ + zero:req,poly:req + + ext v\tmp0\().16b,v\middle\().16b,v\zero\().16b,8 /*high*/ + ext v\middle\().16b,v\zero\().16b,v\middle\().16b,8 /*low */ + eor v\high\().16b,v\high\().16b,v\tmp0\().16b + eor v\low\().16b,v\low\().16b,v\middle\().16b + + pmull2 v\middle\().1q,v\high\().2d,v\poly\().2d + + ext v\tmp0\().16b,v\middle\().16b,v\zero\().16b,8 /*high*/ + ext v\middle\().16b,v\zero\().16b,v\middle\().16b,8 /*low*/ + eor v\high\().16b,v\high\().16b,v\tmp0\().16b + eor v\low\().16b,v\low\().16b,v\middle\().16b + pmull v\middle\().1q,v\high\().1d,v\poly\().1d + eor v\tmp0\().16b, v\low\().16b, v\middle\().16b + eor v\aadhash\().16b, v\aadhash\().16b, v\tmp0\().16b +.endm +.macro ghash_reset_hashkey_addr hashkey_addr:req,hashkey_base:req,count:req + add \hashkey_addr,\hashkey_base,(24-\count)<<5 +.endm + + +.macro ghash_block_n count:req,aadhash:req, dat:req,dat_addr:req, hashkey_addr:req, hashkey_base:req, \ + hashkey:req,hashkey_ext:req,high:req,low:req,middle:req, zero:req,poly:req, \ + tmp0:req,tmp1:req + + ghash_reset_hashkey_addr \hashkey_addr,\hashkey_base,\count + ghash_mult_init_round \aadhash,\dat_addr,\hashkey_addr,\hashkey,\hashkey_ext, \ + \high,\low,\middle,\tmp0,\dat,\count + .set left_count,\count - 1 + .rept left_count + ghash_mult_round \aadhash,\dat_addr,\hashkey_addr,\hashkey,\hashkey_ext, \ + \high,\low,\middle,\tmp0,\tmp1,\dat, left_count + .set left_count,left_count - 1 + + .endr + ghash_mult_final_round \aadhash,\high,\low,\middle,\tmp0,\zero,\poly +.endm + +/* + aadhash=aadhash*[hashkey,hashkey_ext] + rbit(dat) +*/ +.macro ghash_block_reg aadhash:req, dat:req, \ + hashkey:req,hashkey_ext:req,high:req,low:req,middle:req, zero:req,poly:req, \ + tmp0:req + pmull v\middle\().1q,v\aadhash\().1d,v\hashkey_ext\().1d + pmull2 v\tmp0\().1q,v\aadhash\().2d,v\hashkey_ext\().2d + pmull2 v\high\().1q,v\aadhash\().2d,v\hashkey\().2d + eor v\middle\().16b,v\middle\().16b,v\tmp0\().16b + pmull v\low\().1q,v\aadhash\().1d,v\hashkey\().1d + rbit v\aadhash\().16b, v\dat\().16b + ghash_mult_final_round \aadhash,\high,\low,\middle,\tmp0,\zero,\poly +.endm + +.macro ghash_mult_round_noload aadhash:req, \ + hashkey0:req,hashkey0_ext:req,high:req,low:req,middle:req, \ + tmp0:req,tmp1:req + + pmull v\tmp0\().1q,v\aadhash\().1d,v\hashkey0_ext\().1d + pmull2 v\tmp1\().1q,v\aadhash\().2d,v\hashkey0_ext\().2d + eor v\middle\().16b,v\middle\().16b,v\tmp0\().16b + pmull2 v\tmp0\().1q,v\aadhash\().2d,v\hashkey0\().2d + eor v\middle\().16b,v\middle\().16b,v\tmp1\().16b + pmull v\tmp1\().1q,v\aadhash\().1d,v\hashkey0\().1d + eor v\high\().16b,v\high\().16b,v\tmp0\().16b + eor v\low\().16b,v\low\().16b,v\tmp1\().16b + +.endm + +/* aadhash=reduction([low,high],poly)+dat0 */ +.macro poly_mult_final_x2 aadhash:req, \ + high:req,low:req,tmp0:req,tmp1:req, \ + poly:req + pmull2 v\tmp1\().1q,v\high\().2d,v\poly\().2d + eor v\low\().16b, v\aadhash\().16b, v\low\().16b + eor v\aadhash\().16b,v\aadhash\().16b,v\aadhash\().16b + ext v\tmp0\().16b,v\tmp1\().16b,v\aadhash\().16b,8 //high + ext v\tmp1\().16b,v\aadhash\().16b,v\tmp1\().16b,8 //low + eor v\high\().16b,v\high\().16b,v\tmp0\().16b + eor v\low\().16b,v\low\().16b,v\tmp1\().16b + pmull v\tmp1\().1q,v\high\().1d,v\poly\().1d + eor v\aadhash\().16b, v\low\().16b, v\tmp1\().16b +.endm + +.macro aes_encrypt_round block,key + aese v\block\().16b,v\key\().16b + aesmc v\block\().16b,v\block\().16b +.endm + +.macro declare_var_vector_reg name:req,reg:req + q\name .req q\reg + v\name .req v\reg + s\name .req s\reg + d\name .req d\reg +.endm + +.macro declare_var_generic_reg name:req,reg:req + \name .req x\reg + x\name .req x\reg + w\name .req w\reg +.endm + +/*Read data less than 16 */ +.macro read_small_data dest:req,src:req,size:req,tbl_adr:req,tbl:req + ldr q\tbl,[\tbl_adr,\size,lsl 4] + tbz \size,3,1f + ld1 {v\dest\().d}[0],[\src],8 +1: + tbz \size,2,1f + ld1 {v\dest\().s}[2],[\src],4 +1: + tbz \size,1,1f + ld1 {v\dest\().h}[6],[\src],2 +1: + tbz \size,0,1f + ld1 {v\dest\().b}[14],[\src],1 +1: + tbl v\dest\().16b,{v\dest\().16b},v\tbl\().16b +.endm +.macro read_small_data_start dest:req,src:req,size:req,tbl_adr:req,tbl:req + adrp \tbl_adr,:got:read_small_data_table + ldr \tbl_adr,[\tbl_adr,#:got_lo12:read_small_data_table] + read_small_data \dest,\src,\size,\tbl_adr,\tbl +.endm + +.macro read_small_data_end dest:req,src:req,size:req,tbl_adr:req,tbl:req + adrp \tbl_adr,:got:read_end_small_data_table + ldr \tbl_adr,[\tbl_adr,#:got_lo12:read_end_small_data_table] + read_small_data \dest,\src,\size,\tbl_adr,\tbl +.endm + +.macro write_small_data src:req,dest:req,size:req,tbl_adr:req,tmp1:req + ldr q\tmp1,[\tbl_adr,\size,lsl 4] + tbl v\tmp1\().16b,{v\src\().16b},v\tmp1\().16b + tbz \size,3,1f + st1 {v\tmp1\().d}[0],[\dest],8 +1: + tbz \size,2,1f + st1 {v\tmp1\().s}[2],[\dest],4 +1: + tbz \size,1,1f + st1 {v\tmp1\().h}[6],[\dest],2 +1: + tbz \size,0,1f + st1 {v\tmp1\().b}[14],[\dest],1 +1: +.endm +.macro write_small_data_start src:req,dest:req,size:req,tbl_adr:req,tmp1:req + adrp \tbl_adr,:got:write_small_data_table + ldr \tbl_adr,[\tbl_adr,#:got_lo12:write_small_data_table] + write_small_data \src,\dest,\size,\tbl_adr,\tmp1 +.endm +.macro write_small_data_end src:req,dest:req,size:req,tbl_adr:req,tmp1:req + adrp \tbl_adr,:got:write_end_small_data_table + ldr \tbl_adr,[\tbl_adr,#:got_lo12:write_end_small_data_table] + write_small_data \src,\dest,\size,\tbl_adr,\tmp1 +.endm + +.macro tbx_small_data_end src:req,dest:req,size:req,tbl_adr:req,tmp1:req + adrp \tbl_adr,:got:tbx_end_small_data_table + ldr \tbl_adr,[\tbl_adr,#:got_lo12:tbx_end_small_data_table] + ldr q\tmp1,[\tbl_adr,\size,lsl 4] + tbx v\dest\().16b,{v\src\().16b},v\tmp1\().16b +.endm + +.macro tbx_small_data_start src:req,dest:req,size:req,tbl_adr:req,tmp1:req + adrp \tbl_adr,:got:tbx_start_small_data_table + ldr \tbl_adr,[\tbl_adr,#:got_lo12:tbx_start_small_data_table] + ldr q\tmp1,[\tbl_adr,\size,lsl 4] + tbx v\dest\().16b,{v\src\().16b},v\tmp1\().16b +.endm + + +.macro clear_small_data dest:req,zero:req,size:req,tbl_adr:req,tmp1:req + adrp \tbl_adr,:got:shift_small_data_table + ldr \tbl_adr,[\tbl_adr,#:got_lo12:shift_small_data_table] + add \tbl_adr,\tbl_adr,16 + sub \tbl_adr,\tbl_adr,\size + ldr q\tmp1,[\tbl_adr] + tbx v\dest\().16b,{v\zero\().16b},v\tmp1\().16b +.endm + + +.macro aes_gcm_n_round is_enc:req,count:req,aadhash:req, dat_addr:req, \ + hashkey_addr:req, hashkey_base:req, \ + hashkey:req,hashkey_ext:req,high:req,low:req, poly:req, \ + ctr:req,enc_ctr:req,one:req,out_adr:req, \ + tmp0:req,tmp1:req + + ghash_reset_hashkey_addr \hashkey_addr,\hashkey_base,\count + + aes_gcm_init \is_enc,\aadhash,\dat_addr,\hashkey_addr, \ + \hashkey,\hashkey_ext, \high,\low, \ + \ctr,\enc_ctr,\one,\out_adr, \ + \tmp0,\tmp1,\count + + .set left_count,\count - 1 + .rept left_count + aes_gcm_middle \is_enc,\aadhash,\dat_addr,\hashkey_addr, \ + \hashkey,\hashkey_ext, \high,\low, \ + \ctr,\enc_ctr,\one,\out_adr, \ + \tmp0,\tmp1, left_count + .set left_count,left_count - 1 + .endr + + poly_mult_final_x2 \aadhash,\high,\low,\tmp0,\tmp1,\poly + +.endm + + +/* + aadhash=aadhash*[hashkey_base[(TOTAL_HASHKEY_NUM-2),(TOTAL_HASHKEY_NUM-1)]] + rbit(dat) +*/ +.macro ghash_block_reg_x2 aadhash:req, dat:req, hashkey_base:req, \ + hashkey:req,high:req,low:req,tmp0:req, tmp1:req, \ + tmp2:req,temp0:req + ldr q\hashkey,[\hashkey_base,(TOTAL_HASHKEY_NUM-1)*32+16] + eor v\tmp2\().16b,v\tmp2\().16b,v\tmp2\().16b,8 //zero + pmull v\tmp1\().1q,v\aadhash\().1d,v\hashkey\().1d + pmull2 v\tmp0\().1q,v\aadhash\().2d,v\hashkey\().2d + ldr q\hashkey,[\hashkey_base,(TOTAL_HASHKEY_NUM-1)*32] + eor v\tmp0\().16b,v\tmp1\().16b,v\tmp0\().16b + ext v\tmp0\().16b,v\tmp0\().16b,v\tmp2\().16b,8 /*high*/ + ext v\tmp1\().16b,v\tmp2\().16b,v\tmp0\().16b,8 /*low*/ + pmull2 v\high\().1q,v\aadhash\().2d,v\hashkey\().2d + mov temp0,0x87 + pmull v\low\().1q,v\aadhash\().1d,v\hashkey\().1d + dup v\tmp2\().2d,x0 + eor v\high\().16b,v\high\().16b,v\tmp0\().16b + eor v\low\().16b,v\low\().16b,v\tmp1\().16b + rbit v\aadhash\().16b, v\dat\().16b + poly_mult_final_x2 \aadhash,\high,\low,\tmp0,\tmp1,\tmp2 +.endm + +.macro __generic_load_small_data is_enc:req,len_bit:req,small_read_len:req, \ + in_adr:req,out_adr:req,partial_block:req,temp0:req,temp1:req,r:req,p + tbz \small_read_len,\len_bit,1f + ldr\p \r\()\temp0,[\in_adr],1<<\len_bit /*in */ + ldr\p \r\()\temp1,[\partial_block] /* partial*/ + eor \r\()\temp1,\r\()\temp0,\r\()\temp1 + .ifc \is_enc ,decrypt + str\p \r\()\temp0,[\partial_block],1<<\len_bit + .endif + .ifc \is_enc, encrypt + str\p \r\()\temp1,[\partial_block],1<<\len_bit + .endif + str\p \r\()\temp1,[\out_adr],1<<\len_bit +1: +.endm +.macro generic_load_partial_block is_enc:req,small_read_len:req,in_adr:req,out_adr:req, \ + partial_block:req,temp0:req,temp1:req + __generic_load_small_data \is_enc,3,\small_read_len,\in_adr,\out_adr,\partial_block,\temp0,\temp1,x /* small_read_len >=8 */ + __generic_load_small_data \is_enc,2,\small_read_len,\in_adr,\out_adr,\partial_block,\temp0,\temp1,w /* small_read_len >=4 */ + __generic_load_small_data \is_enc,1,\small_read_len,\in_adr,\out_adr,\partial_block,\temp0,\temp1,w,h /* small_read_len >=2 */ + __generic_load_small_data \is_enc,0,\small_read_len,\in_adr,\out_adr,\partial_block,\temp0,\temp1,w,b /* small_read_len >=1 */ +.endm +/* without Neon read version */ +.macro generic_partial_block_start is_enc:req,in_len:req,in_adr:req,out_adr:req,context:req, \ + partial_block:req,partial_block_len:req,small_read_len:req,left_partial_block_len:req, \ + temp0:req + mov \left_partial_block_len,16 + add \partial_block,\context,PARTIAL_BLOCK_ENC_KEY_OFF + sub \left_partial_block_len,\left_partial_block_len,\partial_block_len + add \partial_block,\partial_block,\partial_block_len + cmp \in_len,\left_partial_block_len + csel \small_read_len,\in_len,\left_partial_block_len, ls + add \partial_block_len,\partial_block_len,\small_read_len + sub \in_len,\in_len,\small_read_len + and \partial_block_len,\partial_block_len,0xf + str \partial_block_len,[\context,PARTIAL_BLOCK_LENGTH_OFF] + generic_load_partial_block \is_enc,\small_read_len,\in_adr,\out_adr,\partial_block, \ + \left_partial_block_len,\temp0 /* small_read_len >=8 */ +.endm +.macro generic_paritial_block_end is_enc:req,in_len:req,in_adr:req,out_adr:req,context:req, \ + partial_block:req,temp0:req,temp1:req + str \in_len,[\context,PARTIAL_BLOCK_LENGTH_OFF] + add \partial_block,\context,PARTIAL_BLOCK_ENC_KEY_OFF + generic_load_partial_block \is_enc,\in_len,\in_adr,\out_adr,\partial_block,\temp0,\temp1 /* small_read_len >=8 */ +.endm +/*partial_block_len+in_len < 16,partial_block_len=0,in_len>0 */ +.macro paritial_block_small_length is_enc:req,context:req,in_len:req,in_adr:req,out_adr:req,temp0:req,temp1:req,Ctr:req + + cbz 1f + ldr \temp0,[\context,PARTIAL_BLOCK_LENGTH_OFF] + add \temp1,\temp0,\in_len + str \temp1,[\context,PARTIAL_BLOCK_LENGTH_OFF] + add \context,\temp0,PARTIAL_BLOCK_ENC_KEY_OFF +2:/* loop start */ + sub \in_len,\in_len,1 + ldrb w\temp0,[\in_adr],1 + ldrb w\temp1,[\context] + eor w\temp1,w\temp1,w\temp0 + strb w\temp1,[\out_adr],1 +.ifc \is_enc , encrypt + strb w\temp1,[\context],1 +.endif +.ifc \is_enc,decrypt + strb w\temp0,[\context],1 +.endif + cbnz \in_len,2b +1:/* loop end */ +.endm + +/* 0<in_len < 16,partial_block_len=0 */ +.macro paritial_block_end is_enc:req,context:req,in_len:req,in_adr:req,out_adr:req, \ + temp0:req,partial_block_len:req \ + PartialBlock:req,ctr:req,one:req,Tmp2:req,Tmp3:req,Tmp4:req + add v\ctr\().4s,v\ctr\().4s,v\one\().4s //increase ctr + str q\ctr,[context,CTR_OFF] + read_small_data_start \PartialBlock,\in_adr,\in_len,\tbl_adr,\Tmp0 + aes_encrypt_block \ctr + +.endm + declare_var_vector_reg Key0 ,16 + declare_var_vector_reg Key1 ,17 + declare_var_vector_reg Key2 ,18 + declare_var_vector_reg Key3 ,19 + declare_var_vector_reg Key4 ,20 + declare_var_vector_reg Key5 ,21 + declare_var_vector_reg Key6 ,22 + declare_var_vector_reg Key7 ,23 + declare_var_vector_reg Key8 ,24 + declare_var_vector_reg Key9 ,25 + declare_var_vector_reg Key10,26 |