Adding upstream version 18.2.2.upstream/18.2.2

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 430 insertions, 0 deletions

diff --git a/src/crypto/isa-l/isa-l_crypto/aes/aarch64/gcm_common.S b/src/crypto/isa-l/isa-l_crypto/aes/aarch64/gcm_common.S
new file mode 100644
index 000000000..042f6cf19
--- /dev/null
+++ b/src/crypto/isa-l/isa-l_crypto/aes/aarch64/gcm_common.S
@@ -0,0 +1,430 @@
+/**********************************************************************
+  Copyright(c) 2021 Arm Corporation All rights reserved.
+
+  Redistribution and use in source and binary forms, with or without
+  modification, are permitted provided that the following conditions
+  are met:
+    * Redistributions of source code must retain the above copyright
+      notice, this list of conditions and the following disclaimer.
+    * Redistributions in binary form must reproduce the above copyright
+      notice, this list of conditions and the following disclaimer in
+      the documentation and/or other materials provided with the
+      distribution.
+    * Neither the name of Arm Corporation nor the names of its
+      contributors may be used to endorse or promote products derived
+      from this software without specific prior written permission.
+
+  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+  OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+**********************************************************************/
+    .arch armv8-a+crypto
+    .text
+#define HASHKEY_TOTAL_NUM       (24)
+#define HASHKEY_BASE_OFF        (15*16)
+#define HASHKEY_OFF(n)          ((15*16)+n*32)
+#define HASHKEY_EXT_OFF(n)      ((15*16)+n*32+16)
+#ifndef KEY_LEN
+#define KEY_LEN 128
+#endif
+#ifndef BLOCKS
+#define BLOCKS 24
+#endif
+#define FN_NAME(fn,mode,post) aes_gcm_##fn##_##mode####post##aes
+#define START_FUNC(fn,mode,post) .global FN_NAME(fn,mode,post); \
+    .type   FN_NAME(fn,mode,post), %function; \
+    FN_NAME(fn,mode,post):
+#define END_FUNC(fn,mode,post)   .size  FN_NAME(fn,mode,post), .-FN_NAME(fn,mode,post)
+
+#define AAD_LEN_OFF                 16
+#define IN_LENGTH_OFF               24
+#define PARTIAL_BLOCK_ENC_KEY_OFF   32
+#define PARTIAL_BLOCK_LENGTH_OFF    80
+#define CTR_OFF                     64
+#define ORIG_IV_OFF                 48
+/*
+    [low,middle,tmp0,high] +=dat0 * [hashkey0,hashkey0_ext]
+    ifnb dat1
+        dat1=rbit(*dat_adr)
+        [hashkey0,hashkey0_ext] = *hashkey_adr
+        dat_adr+=16
+        hashkey_adr+=32
+*/
+
+.macro  ghash_mult_round    aadhash:req,dat_adr:req,hashkey_adr:req,   \
+    hashkey0:req,hashkey0_ext:req,high:req,low:req,middle:req,         \
+    tmp0:req,tmp1:req,next_dat:req,left_count:req
+
+    ldr      q\next_dat,[\dat_adr],16
+    pmull    v\tmp0\().1q,v\aadhash\().1d,v\hashkey0_ext\().1d
+    pmull2   v\tmp1\().1q,v\aadhash\().2d,v\hashkey0_ext\().2d
+    .if  \left_count > 1
+        ldr      q\hashkey0_ext,[\hashkey_adr,16]
+    .endif
+    eor      v\middle\().16b,v\middle\().16b,v\tmp0\().16b
+    pmull2   v\tmp0\().1q,v\aadhash\().2d,v\hashkey0\().2d
+    eor      v\middle\().16b,v\middle\().16b,v\tmp1\().16b
+    pmull    v\tmp1\().1q,v\aadhash\().1d,v\hashkey0\().1d
+    .if  \left_count > 1
+        ldr      q\hashkey0,[\hashkey_adr],32
+    .endif
+    eor      v\high\().16b,v\high\().16b,v\tmp0\().16b
+    eor      v\low\().16b,v\low\().16b,v\tmp1\().16b
+    rbit     v\aadhash\().16b, v\next_dat\().16b
+.endm
+
+.macro  ghash_mult_init_round    aadhash:req,dat_adr:req,hashkey_adr:req,   \
+    hashkey0:req,hashkey0_ext:req,                                          \
+    high:req,low:req,middle:req,tmp0:req,next_dat:req,left_count:req
+    ldp     q\hashkey0,q\hashkey0_ext,[\hashkey_adr],32
+    ldr      q\next_dat,[\dat_adr],16
+    pmull    v\middle\().1q,v\aadhash\().1d,v\hashkey0_ext\().1d
+    pmull2   v\tmp0\().1q,v\aadhash\().2d,v\hashkey0_ext\().2d
+    .if  \left_count > 1
+        ldr      q\hashkey0_ext,[\hashkey_adr,16]
+    .endif
+    pmull2   v\high\().1q,v\aadhash\().2d,v\hashkey0\().2d
+    eor      v\middle\().16b,v\middle\().16b,v\tmp0\().16b
+
+    pmull    v\low\().1q,v\aadhash\().1d,v\hashkey0\().1d
+    .if  \left_count > 1
+        ldr      q\hashkey0,[\hashkey_adr],32
+    .endif
+    rbit     v\aadhash\().16b, v\next_dat\().16b
+.endm
+
+/* aadhash=reduction(low,middle,high)+dat0 */
+.macro  ghash_mult_final_round aadhash:req,     \
+    high:req,low:req,middle:req,tmp0:req,       \
+    zero:req,poly:req
+
+    ext      v\tmp0\().16b,v\middle\().16b,v\zero\().16b,8      /*high*/
+    ext      v\middle\().16b,v\zero\().16b,v\middle\().16b,8    /*low */
+    eor      v\high\().16b,v\high\().16b,v\tmp0\().16b
+    eor      v\low\().16b,v\low\().16b,v\middle\().16b
+
+    pmull2   v\middle\().1q,v\high\().2d,v\poly\().2d
+
+    ext      v\tmp0\().16b,v\middle\().16b,v\zero\().16b,8      /*high*/
+    ext      v\middle\().16b,v\zero\().16b,v\middle\().16b,8    /*low*/
+    eor      v\high\().16b,v\high\().16b,v\tmp0\().16b
+    eor      v\low\().16b,v\low\().16b,v\middle\().16b
+    pmull    v\middle\().1q,v\high\().1d,v\poly\().1d
+    eor      v\tmp0\().16b, v\low\().16b, v\middle\().16b
+    eor      v\aadhash\().16b, v\aadhash\().16b, v\tmp0\().16b
+.endm
+.macro  ghash_reset_hashkey_addr    hashkey_addr:req,hashkey_base:req,count:req
+    add     \hashkey_addr,\hashkey_base,(24-\count)<<5
+.endm
+
+
+.macro ghash_block_n count:req,aadhash:req, dat:req,dat_addr:req, hashkey_addr:req, hashkey_base:req, \
+    hashkey:req,hashkey_ext:req,high:req,low:req,middle:req, zero:req,poly:req, \
+    tmp0:req,tmp1:req
+
+    ghash_reset_hashkey_addr    \hashkey_addr,\hashkey_base,\count
+    ghash_mult_init_round   \aadhash,\dat_addr,\hashkey_addr,\hashkey,\hashkey_ext, \
+        \high,\low,\middle,\tmp0,\dat,\count
+    .set left_count,\count - 1
+    .rept left_count
+        ghash_mult_round        \aadhash,\dat_addr,\hashkey_addr,\hashkey,\hashkey_ext, \
+            \high,\low,\middle,\tmp0,\tmp1,\dat, left_count
+        .set left_count,left_count - 1
+
+    .endr
+    ghash_mult_final_round  \aadhash,\high,\low,\middle,\tmp0,\zero,\poly
+.endm
+
+/*
+    aadhash=aadhash*[hashkey,hashkey_ext] + rbit(dat)
+*/
+.macro ghash_block_reg aadhash:req, dat:req, \
+    hashkey:req,hashkey_ext:req,high:req,low:req,middle:req, zero:req,poly:req, \
+    tmp0:req
+    pmull    v\middle\().1q,v\aadhash\().1d,v\hashkey_ext\().1d
+    pmull2   v\tmp0\().1q,v\aadhash\().2d,v\hashkey_ext\().2d
+    pmull2   v\high\().1q,v\aadhash\().2d,v\hashkey\().2d
+    eor      v\middle\().16b,v\middle\().16b,v\tmp0\().16b
+    pmull    v\low\().1q,v\aadhash\().1d,v\hashkey\().1d
+    rbit     v\aadhash\().16b, v\dat\().16b
+    ghash_mult_final_round  \aadhash,\high,\low,\middle,\tmp0,\zero,\poly
+.endm
+
+.macro  ghash_mult_round_noload    aadhash:req,   \
+    hashkey0:req,hashkey0_ext:req,high:req,low:req,middle:req,         \
+    tmp0:req,tmp1:req
+
+    pmull    v\tmp0\().1q,v\aadhash\().1d,v\hashkey0_ext\().1d
+    pmull2   v\tmp1\().1q,v\aadhash\().2d,v\hashkey0_ext\().2d
+    eor      v\middle\().16b,v\middle\().16b,v\tmp0\().16b
+    pmull2   v\tmp0\().1q,v\aadhash\().2d,v\hashkey0\().2d
+    eor      v\middle\().16b,v\middle\().16b,v\tmp1\().16b
+    pmull    v\tmp1\().1q,v\aadhash\().1d,v\hashkey0\().1d
+    eor      v\high\().16b,v\high\().16b,v\tmp0\().16b
+    eor      v\low\().16b,v\low\().16b,v\tmp1\().16b
+
+.endm
+
+/* aadhash=reduction([low,high],poly)+dat0 */
+.macro  poly_mult_final_x2 aadhash:req, \
+    high:req,low:req,tmp0:req,tmp1:req,      \
+    poly:req
+    pmull2   v\tmp1\().1q,v\high\().2d,v\poly\().2d
+    eor      v\low\().16b, v\aadhash\().16b, v\low\().16b
+    eor      v\aadhash\().16b,v\aadhash\().16b,v\aadhash\().16b
+    ext      v\tmp0\().16b,v\tmp1\().16b,v\aadhash\().16b,8 //high
+    ext      v\tmp1\().16b,v\aadhash\().16b,v\tmp1\().16b,8 //low
+    eor      v\high\().16b,v\high\().16b,v\tmp0\().16b
+    eor      v\low\().16b,v\low\().16b,v\tmp1\().16b
+    pmull    v\tmp1\().1q,v\high\().1d,v\poly\().1d
+    eor      v\aadhash\().16b, v\low\().16b, v\tmp1\().16b
+.endm
+
+.macro aes_encrypt_round    block,key
+    aese    v\block\().16b,v\key\().16b
+    aesmc   v\block\().16b,v\block\().16b
+.endm
+
+.macro  declare_var_vector_reg name:req,reg:req
+    q\name      .req    q\reg
+    v\name      .req    v\reg
+    s\name      .req    s\reg
+    d\name      .req    d\reg
+.endm
+
+.macro  declare_var_generic_reg name:req,reg:req
+     \name      .req    x\reg
+    x\name      .req    x\reg
+    w\name      .req    w\reg
+.endm
+
+/*Read data less than 16 */
+.macro read_small_data  dest:req,src:req,size:req,tbl_adr:req,tbl:req
+    ldr     q\tbl,[\tbl_adr,\size,lsl 4]
+    tbz     \size,3,1f
+    ld1     {v\dest\().d}[0],[\src],8
+1:
+    tbz     \size,2,1f
+    ld1     {v\dest\().s}[2],[\src],4
+1:
+    tbz     \size,1,1f
+    ld1     {v\dest\().h}[6],[\src],2
+1:
+    tbz     \size,0,1f
+    ld1     {v\dest\().b}[14],[\src],1
+1:
+    tbl     v\dest\().16b,{v\dest\().16b},v\tbl\().16b
+.endm
+.macro read_small_data_start  dest:req,src:req,size:req,tbl_adr:req,tbl:req
+    adrp    \tbl_adr,:got:read_small_data_table
+    ldr     \tbl_adr,[\tbl_adr,#:got_lo12:read_small_data_table]
+    read_small_data \dest,\src,\size,\tbl_adr,\tbl
+.endm
+
+.macro read_small_data_end  dest:req,src:req,size:req,tbl_adr:req,tbl:req
+    adrp    \tbl_adr,:got:read_end_small_data_table
+    ldr     \tbl_adr,[\tbl_adr,#:got_lo12:read_end_small_data_table]
+    read_small_data \dest,\src,\size,\tbl_adr,\tbl
+.endm
+
+.macro write_small_data  src:req,dest:req,size:req,tbl_adr:req,tmp1:req
+    ldr     q\tmp1,[\tbl_adr,\size,lsl 4]
+    tbl     v\tmp1\().16b,{v\src\().16b},v\tmp1\().16b
+    tbz     \size,3,1f
+    st1     {v\tmp1\().d}[0],[\dest],8
+1:
+    tbz     \size,2,1f
+    st1     {v\tmp1\().s}[2],[\dest],4
+1:
+    tbz     \size,1,1f
+    st1     {v\tmp1\().h}[6],[\dest],2
+1:
+    tbz     \size,0,1f
+    st1     {v\tmp1\().b}[14],[\dest],1
+1:
+.endm
+.macro write_small_data_start  src:req,dest:req,size:req,tbl_adr:req,tmp1:req
+    adrp    \tbl_adr,:got:write_small_data_table
+    ldr     \tbl_adr,[\tbl_adr,#:got_lo12:write_small_data_table]
+    write_small_data \src,\dest,\size,\tbl_adr,\tmp1
+.endm
+.macro write_small_data_end  src:req,dest:req,size:req,tbl_adr:req,tmp1:req
+    adrp    \tbl_adr,:got:write_end_small_data_table
+    ldr     \tbl_adr,[\tbl_adr,#:got_lo12:write_end_small_data_table]
+    write_small_data \src,\dest,\size,\tbl_adr,\tmp1
+.endm
+
+.macro tbx_small_data_end   src:req,dest:req,size:req,tbl_adr:req,tmp1:req
+    adrp    \tbl_adr,:got:tbx_end_small_data_table
+    ldr     \tbl_adr,[\tbl_adr,#:got_lo12:tbx_end_small_data_table]
+    ldr     q\tmp1,[\tbl_adr,\size,lsl 4]
+    tbx     v\dest\().16b,{v\src\().16b},v\tmp1\().16b
+.endm
+
+.macro tbx_small_data_start   src:req,dest:req,size:req,tbl_adr:req,tmp1:req
+    adrp    \tbl_adr,:got:tbx_start_small_data_table
+    ldr     \tbl_adr,[\tbl_adr,#:got_lo12:tbx_start_small_data_table]
+    ldr     q\tmp1,[\tbl_adr,\size,lsl 4]
+    tbx     v\dest\().16b,{v\src\().16b},v\tmp1\().16b
+.endm
+
+
+.macro clear_small_data   dest:req,zero:req,size:req,tbl_adr:req,tmp1:req
+    adrp    \tbl_adr,:got:shift_small_data_table
+    ldr     \tbl_adr,[\tbl_adr,#:got_lo12:shift_small_data_table]
+    add     \tbl_adr,\tbl_adr,16
+    sub     \tbl_adr,\tbl_adr,\size
+    ldr     q\tmp1,[\tbl_adr]
+    tbx     v\dest\().16b,{v\zero\().16b},v\tmp1\().16b
+.endm
+
+
+.macro aes_gcm_n_round is_enc:req,count:req,aadhash:req, dat_addr:req,  \
+    hashkey_addr:req, hashkey_base:req,                                 \
+    hashkey:req,hashkey_ext:req,high:req,low:req, poly:req,             \
+    ctr:req,enc_ctr:req,one:req,out_adr:req,                            \
+    tmp0:req,tmp1:req
+
+    ghash_reset_hashkey_addr    \hashkey_addr,\hashkey_base,\count
+
+    aes_gcm_init    \is_enc,\aadhash,\dat_addr,\hashkey_addr,           \
+        \hashkey,\hashkey_ext, \high,\low,                              \
+        \ctr,\enc_ctr,\one,\out_adr,                                    \
+        \tmp0,\tmp1,\count
+
+    .set left_count,\count - 1
+    .rept left_count
+        aes_gcm_middle        \is_enc,\aadhash,\dat_addr,\hashkey_addr, \
+            \hashkey,\hashkey_ext, \high,\low,                          \
+            \ctr,\enc_ctr,\one,\out_adr,                                \
+            \tmp0,\tmp1, left_count
+        .set left_count,left_count - 1
+    .endr
+
+    poly_mult_final_x2  \aadhash,\high,\low,\tmp0,\tmp1,\poly
+
+.endm
+
+
+/*
+    aadhash=aadhash*[hashkey_base[(TOTAL_HASHKEY_NUM-2),(TOTAL_HASHKEY_NUM-1)]] + rbit(dat)
+*/
+.macro ghash_block_reg_x2 aadhash:req, dat:req, hashkey_base:req,       \
+    hashkey:req,high:req,low:req,tmp0:req, tmp1:req,                    \
+    tmp2:req,temp0:req
+    ldr     q\hashkey,[\hashkey_base,(TOTAL_HASHKEY_NUM-1)*32+16]
+    eor     v\tmp2\().16b,v\tmp2\().16b,v\tmp2\().16b,8 //zero
+    pmull    v\tmp1\().1q,v\aadhash\().1d,v\hashkey\().1d
+    pmull2   v\tmp0\().1q,v\aadhash\().2d,v\hashkey\().2d
+    ldr     q\hashkey,[\hashkey_base,(TOTAL_HASHKEY_NUM-1)*32]
+    eor      v\tmp0\().16b,v\tmp1\().16b,v\tmp0\().16b
+    ext      v\tmp0\().16b,v\tmp0\().16b,v\tmp2\().16b,8 /*high*/
+    ext      v\tmp1\().16b,v\tmp2\().16b,v\tmp0\().16b,8 /*low*/
+    pmull2   v\high\().1q,v\aadhash\().2d,v\hashkey\().2d
+    mov      temp0,0x87
+    pmull    v\low\().1q,v\aadhash\().1d,v\hashkey\().1d
+    dup      v\tmp2\().2d,x0
+    eor      v\high\().16b,v\high\().16b,v\tmp0\().16b
+    eor      v\low\().16b,v\low\().16b,v\tmp1\().16b
+    rbit     v\aadhash\().16b, v\dat\().16b
+    poly_mult_final_x2  \aadhash,\high,\low,\tmp0,\tmp1,\tmp2
+.endm
+
+.macro __generic_load_small_data  is_enc:req,len_bit:req,small_read_len:req,  \
+        in_adr:req,out_adr:req,partial_block:req,temp0:req,temp1:req,r:req,p
+    tbz         \small_read_len,\len_bit,1f
+    ldr\p       \r\()\temp0,[\in_adr],1<<\len_bit           /*in */
+    ldr\p       \r\()\temp1,[\partial_block]                /* partial*/
+    eor         \r\()\temp1,\r\()\temp0,\r\()\temp1
+    .ifc \is_enc ,decrypt
+        str\p   \r\()\temp0,[\partial_block],1<<\len_bit
+    .endif
+    .ifc \is_enc, encrypt
+        str\p   \r\()\temp1,[\partial_block],1<<\len_bit
+    .endif
+    str\p   \r\()\temp1,[\out_adr],1<<\len_bit
+1:
+.endm
+.macro  generic_load_partial_block is_enc:req,small_read_len:req,in_adr:req,out_adr:req, \
+    partial_block:req,temp0:req,temp1:req
+    __generic_load_small_data \is_enc,3,\small_read_len,\in_adr,\out_adr,\partial_block,\temp0,\temp1,x     /*  small_read_len >=8 */
+    __generic_load_small_data \is_enc,2,\small_read_len,\in_adr,\out_adr,\partial_block,\temp0,\temp1,w     /*  small_read_len >=4 */
+    __generic_load_small_data \is_enc,1,\small_read_len,\in_adr,\out_adr,\partial_block,\temp0,\temp1,w,h   /*  small_read_len >=2 */
+    __generic_load_small_data \is_enc,0,\small_read_len,\in_adr,\out_adr,\partial_block,\temp0,\temp1,w,b   /*  small_read_len >=1 */
+.endm
+/* without Neon read version */
+.macro  generic_partial_block_start    is_enc:req,in_len:req,in_adr:req,out_adr:req,context:req,   \
+        partial_block:req,partial_block_len:req,small_read_len:req,left_partial_block_len:req,     \
+        temp0:req
+    mov         \left_partial_block_len,16
+    add         \partial_block,\context,PARTIAL_BLOCK_ENC_KEY_OFF
+    sub         \left_partial_block_len,\left_partial_block_len,\partial_block_len
+    add         \partial_block,\partial_block,\partial_block_len
+    cmp         \in_len,\left_partial_block_len
+    csel        \small_read_len,\in_len,\left_partial_block_len, ls
+    add         \partial_block_len,\partial_block_len,\small_read_len
+    sub         \in_len,\in_len,\small_read_len
+    and         \partial_block_len,\partial_block_len,0xf
+    str         \partial_block_len,[\context,PARTIAL_BLOCK_LENGTH_OFF]
+    generic_load_partial_block \is_enc,\small_read_len,\in_adr,\out_adr,\partial_block, \
+        \left_partial_block_len,\temp0     /*  small_read_len >=8 */
+.endm
+.macro  generic_paritial_block_end    is_enc:req,in_len:req,in_adr:req,out_adr:req,context:req,   \
+        partial_block:req,temp0:req,temp1:req
+    str         \in_len,[\context,PARTIAL_BLOCK_LENGTH_OFF]
+    add         \partial_block,\context,PARTIAL_BLOCK_ENC_KEY_OFF
+    generic_load_partial_block \is_enc,\in_len,\in_adr,\out_adr,\partial_block,\temp0,\temp1     /*  small_read_len >=8 */
+.endm
+/*partial_block_len+in_len < 16,partial_block_len=0,in_len>0 */
+.macro  paritial_block_small_length is_enc:req,context:req,in_len:req,in_adr:req,out_adr:req,temp0:req,temp1:req,Ctr:req
+
+    cbz     1f
+    ldr     \temp0,[\context,PARTIAL_BLOCK_LENGTH_OFF]
+    add     \temp1,\temp0,\in_len
+    str     \temp1,[\context,PARTIAL_BLOCK_LENGTH_OFF]
+    add     \context,\temp0,PARTIAL_BLOCK_ENC_KEY_OFF
+2:/* loop start */
+    sub     \in_len,\in_len,1
+    ldrb    w\temp0,[\in_adr],1
+    ldrb    w\temp1,[\context]
+    eor     w\temp1,w\temp1,w\temp0
+    strb    w\temp1,[\out_adr],1
+.ifc \is_enc , encrypt
+    strb    w\temp1,[\context],1
+.endif
+.ifc \is_enc,decrypt
+    strb    w\temp0,[\context],1
+.endif
+    cbnz     \in_len,2b
+1:/* loop end */
+.endm
+
+/* 0<in_len < 16,partial_block_len=0 */
+.macro  paritial_block_end  is_enc:req,context:req,in_len:req,in_adr:req,out_adr:req,   \
+    temp0:req,partial_block_len:req    \
+    PartialBlock:req,ctr:req,one:req,Tmp2:req,Tmp3:req,Tmp4:req
+    add         v\ctr\().4s,v\ctr\().4s,v\one\().4s    //increase ctr
+    str         q\ctr,[context,CTR_OFF]
+    read_small_data_start   \PartialBlock,\in_adr,\in_len,\tbl_adr,\Tmp0
+    aes_encrypt_block   \ctr
+
+.endm
+        declare_var_vector_reg  Key0 ,16
+        declare_var_vector_reg  Key1 ,17
+        declare_var_vector_reg  Key2 ,18
+        declare_var_vector_reg  Key3 ,19
+        declare_var_vector_reg  Key4 ,20
+        declare_var_vector_reg  Key5 ,21
+        declare_var_vector_reg  Key6 ,22
+        declare_var_vector_reg  Key7 ,23
+        declare_var_vector_reg  Key8 ,24
+        declare_var_vector_reg  Key9 ,25
+        declare_var_vector_reg  Key10,26