diff options
Diffstat (limited to 'src/isa-l/crc/aarch64')
40 files changed, 5806 insertions, 0 deletions
diff --git a/src/isa-l/crc/aarch64/Makefile.am b/src/isa-l/crc/aarch64/Makefile.am new file mode 100644 index 000000000..5113b77e9 --- /dev/null +++ b/src/isa-l/crc/aarch64/Makefile.am @@ -0,0 +1,57 @@ +######################################################################## +# Copyright(c) 2020 Arm Corporation All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Arm Corporation nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +######################################################################### + +lsrc_aarch64 += \ + crc/aarch64/crc_multibinary_arm.S \ + crc/aarch64/crc_aarch64_dispatcher.c + +lsrc_aarch64 += \ + crc/aarch64/crc16_t10dif_pmull.S \ + crc/aarch64/crc16_t10dif_copy_pmull.S \ + crc/aarch64/crc32_ieee_norm_pmull.S \ + crc/aarch64/crc64_ecma_refl_pmull.S \ + crc/aarch64/crc64_ecma_norm_pmull.S \ + crc/aarch64/crc64_iso_refl_pmull.S \ + crc/aarch64/crc64_iso_norm_pmull.S \ + crc/aarch64/crc64_jones_refl_pmull.S \ + crc/aarch64/crc64_jones_norm_pmull.S + +#CRC32/CRC32C for micro-architecture +lsrc_aarch64 += \ + crc/aarch64/crc32_iscsi_refl_pmull.S \ + crc/aarch64/crc32_gzip_refl_pmull.S \ + crc/aarch64/crc32_iscsi_3crc_fold.S \ + crc/aarch64/crc32_gzip_refl_3crc_fold.S \ + crc/aarch64/crc32_iscsi_crc_ext.S \ + crc/aarch64/crc32_gzip_refl_crc_ext.S \ + crc/aarch64/crc32_mix_default.S \ + crc/aarch64/crc32c_mix_default.S \ + crc/aarch64/crc32_mix_neoverse_n1.S \ + crc/aarch64/crc32c_mix_neoverse_n1.S + diff --git a/src/isa-l/crc/aarch64/crc16_t10dif_copy_pmull.S b/src/isa-l/crc/aarch64/crc16_t10dif_copy_pmull.S new file mode 100644 index 000000000..10bf157c2 --- /dev/null +++ b/src/isa-l/crc/aarch64/crc16_t10dif_copy_pmull.S @@ -0,0 +1,423 @@ +######################################################################## +# Copyright(c) 2019 Arm Corporation All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Arm Corporation nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +######################################################################### + + .arch armv8-a+crc+crypto + .text + .align 3 + .global crc16_t10dif_copy_pmull + .type crc16_t10dif_copy_pmull, %function + +/* uint16_t crc16_t10dif_pmull(uint16_t seed, uint8_t *buf, uint64_t len) */ + +/* arguments */ +w_seed .req w0 +x_dst .req x1 +x_src .req x2 +x_len .req x3 +w_len .req w3 + +/* returns */ +w_ret .req w0 + +/* these as global temporary registers */ +w_tmp .req w6 +x_tmp .req x6 +x_tmp1 .req x7 +x_tmp2 .req x11 + +d_tmp1 .req d0 +d_tmp2 .req d1 +q_tmp1 .req q0 +q_tmp2 .req q1 +v_tmp1 .req v0 +v_tmp2 .req v1 + +/* local variables */ +w_counter .req w4 +w_crc .req w0 +x_crc .req x0 +x_counter .req x4 +x_crc16tab .req x5 +x_src_saved .req x0 +x_dst_saved .req x12 + +crc16_t10dif_copy_pmull: + cmp x_len, 1023 + sub sp, sp, #16 + uxth w_seed, w_seed + bhi .crc_fold + + mov x_tmp, 0 + mov w_counter, 0 + +.crc_table_loop_pre: + cmp x_len, x_tmp + bls .end + + sxtw x_counter, w_counter + adrp x_crc16tab, .LANCHOR0 + sub x_src, x_src, x_counter + sub x_dst, x_dst, x_counter + add x_crc16tab, x_crc16tab, :lo12:.LANCHOR0 + + .align 2 +.crc_table_loop: + ldrb w_tmp, [x_src, x_counter] + strb w_tmp, [x_dst, x_counter] + add x_counter, x_counter, 1 + cmp x_len, x_counter + eor w_tmp, w_tmp, w_crc, lsr 8 + ldrh w_tmp, [x_crc16tab, w_tmp, sxtw 1] + eor w_crc, w_tmp, w_crc, lsl 8 + uxth w_crc, w_crc + bhi .crc_table_loop + +.end: + add sp, sp, 16 + ret + +/* carry less multiplication, part1 - before loop */ +q_x0 .req q2 +q_x1 .req q3 +q_x2 .req q4 +q_x3 .req q5 + +v_x0 .req v2 +v_x1 .req v3 +v_x2 .req v4 +v_x3 .req v5 + +d_x0 .req d2 +d_x1 .req d3 +d_x2 .req d4 +d_x3 .req d5 + +// the following registers only used this part1 +d_tmp3 .req d16 +v_tmp3 .req v16 + + .align 3 +.crc_fold: + fmov d_tmp1, x_crc + fmov d_tmp2, xzr + dup d_tmp3, v_tmp2.d[0] + shl d_tmp1, d_tmp1, 48 + ins v_tmp3.d[1], v_tmp1.d[0] + + and x_counter, x_len, -64 + sub x_counter, x_counter, #64 + cmp x_counter, 63 + add x_src_saved, x_src, 64 + add x_dst_saved, x_dst, 64 + + ldr q_x0, [x_src] + ldr q_x1, [x_src, 16] + ldr q_x2, [x_src, 32] + ldr q_x3, [x_src, 48] + + str q_x0, [x_dst] + str q_x1, [x_dst, 16] + str q_x2, [x_dst, 32] + str q_x3, [x_dst, 48] + + adrp x_tmp, .shuffle_mask_lanchor + ldr q7, [x_tmp, :lo12:.shuffle_mask_lanchor] + + tbl v_tmp1.16b, {v_x0.16b}, v7.16b + eor v_x0.16b, v_tmp3.16b, v_tmp1.16b + + tbl v_x1.16b, {v_x1.16b}, v7.16b + tbl v_x2.16b, {v_x2.16b}, v7.16b + tbl v_x3.16b, {v_x3.16b}, v7.16b + bls .crc_fold_loop_end + +/* carry less multiplication, part2 - loop */ +q_y0 .req q28 +q_y1 .req q29 +q_y2 .req q30 +q_y3 .req q31 + +v_y0 .req v28 +v_y1 .req v29 +v_y2 .req v30 +v_y3 .req v31 + +d_x0_h .req d24 +d_x0_l .req d2 +d_x1_h .req d25 +d_x1_l .req d3 +d_x2_h .req d26 +d_x2_l .req d4 +d_x3_h .req d27 +d_x3_l .req d5 + +v_x0_h .req v24 +v_x0_l .req v2 +v_x1_h .req v25 +v_x1_l .req v3 +v_x2_h .req v26 +v_x2_l .req v4 +v_x3_h .req v27 +v_x3_l .req v5 + +v_tmp1_x0 .req v24 +v_tmp1_x1 .req v25 +v_tmp1_x2 .req v26 +v_tmp1_x3 .req v27 + +d_p4_h .req d19 +v_p4_h .req v19 +d_p4_l .req d17 +v_p4_l .req v17 + + mov x_tmp, 0x371d0000 /* p4 [1] */ + fmov d_p4_h, x_tmp + mov x_tmp, 0x87e70000 /* p4 [0] */ + fmov d_p4_l, x_tmp + + .align 2 +.crc_fold_loop: + add x_src_saved, x_src_saved, 64 + add x_dst_saved, x_dst_saved, 64 + + sub x_counter, x_counter, #64 + cmp x_counter, 63 + + dup d_x0_h, v_x0.d[1] + dup d_x1_h, v_x1.d[1] + dup d_x2_h, v_x2.d[1] + dup d_x3_h, v_x3.d[1] + + dup d_x0_l, v_x0.d[0] + dup d_x1_l, v_x1.d[0] + dup d_x2_l, v_x2.d[0] + dup d_x3_l, v_x3.d[0] + + ldr q_y0, [x_src_saved, -64] + ldr q_y1, [x_src_saved, -48] + ldr q_y2, [x_src_saved, -32] + ldr q_y3, [x_src_saved, -16] + + str q_y0, [x_dst_saved, -64] + str q_y1, [x_dst_saved, -48] + str q_y2, [x_dst_saved, -32] + str q_y3, [x_dst_saved, -16] + + pmull v_x0_h.1q, v_x0_h.1d, v_p4_h.1d + pmull v_x0_l.1q, v_x0_l.1d, v_p4_l.1d + pmull v_x1_h.1q, v_x1_h.1d, v_p4_h.1d + pmull v_x1_l.1q, v_x1_l.1d, v_p4_l.1d + pmull v_x2_h.1q, v_x2_h.1d, v_p4_h.1d + pmull v_x2_l.1q, v_x2_l.1d, v_p4_l.1d + pmull v_x3_h.1q, v_x3_h.1d, v_p4_h.1d + pmull v_x3_l.1q, v_x3_l.1d, v_p4_l.1d + + tbl v_y0.16b, {v_y0.16b}, v7.16b + tbl v_y1.16b, {v_y1.16b}, v7.16b + tbl v_y2.16b, {v_y2.16b}, v7.16b + tbl v_y3.16b, {v_y3.16b}, v7.16b + + eor v_tmp1_x0.16b, v_x0_h.16b, v_x0_l.16b + eor v_tmp1_x1.16b, v_x1_h.16b, v_x1_l.16b + eor v_tmp1_x2.16b, v_x2_h.16b, v_x2_l.16b + eor v_tmp1_x3.16b, v_x3_h.16b, v_x3_l.16b + + eor v_x0.16b, v_tmp1_x0.16b, v_y0.16b + eor v_x1.16b, v_tmp1_x1.16b, v_y1.16b + eor v_x2.16b, v_tmp1_x2.16b, v_y2.16b + eor v_x3.16b, v_tmp1_x3.16b, v_y3.16b + + bhi .crc_fold_loop + +/* carry less multiplication, part3 - after loop */ +/* folding 512bit ---> 128bit */ + +// input parameters: +// v_x0 => v2 +// v_x1 => v3 +// v_x2 => v4 +// v_x3 => v5 + +// v0, v1, v6, v30, are tmp registers + +.crc_fold_loop_end: + mov x_tmp, 0x4c1a0000 /* p1 [1] */ + fmov d0, x_tmp + mov x_tmp, 0xfb0b0000 /* p1 [0] */ + fmov d1, x_tmp + + and w_counter, w_len, -64 + sxtw x_tmp, w_counter + + add x_src, x_src, x_tmp + add x_dst, x_dst, x_tmp + + dup d6, v_x0.d[1] + dup d30, v_x0.d[0] + pmull v6.1q, v6.1d, v0.1d + pmull v30.1q, v30.1d, v1.1d + eor v6.16b, v6.16b, v30.16b + eor v_x1.16b, v6.16b, v_x1.16b + + dup d6, v_x1.d[1] + dup d30, v_x1.d[0] + pmull v6.1q, v6.1d, v0.1d + pmull v16.1q, v30.1d, v1.1d + eor v6.16b, v6.16b, v16.16b + eor v_x2.16b, v6.16b, v_x2.16b + + dup d_x0, v_x2.d[1] + dup d30, v_x2.d[0] + pmull v0.1q, v_x0.1d, v0.1d + pmull v_x0.1q, v30.1d, v1.1d + eor v1.16b, v0.16b, v_x0.16b + eor v_x0.16b, v1.16b, v_x3.16b + +/* carry less multiplication, part3 - after loop */ +/* crc16 fold function */ +d_16fold_p0_h .req d18 +v_16fold_p0_h .req v18 + +d_16fold_p0_l .req d4 +v_16fold_p0_l .req v4 + +v_16fold_from .req v_x0 +d_16fold_from_h .req d3 +v_16fold_from_h .req v3 + +v_16fold_zero .req v7 + +v_16fold_from1 .req v16 + +v_16fold_from2 .req v0 +d_16fold_from2_h .req d6 +v_16fold_from2_h .req v6 + +v_16fold_tmp .req v0 + + movi v_16fold_zero.4s, 0 + mov x_tmp1, 0x2d560000 /* p0 [1] */ + mov x_tmp2, 0x13680000 /* p0 [0] */ + + ext v_16fold_tmp.16b, v_16fold_zero.16b, v_16fold_from.16b, #8 + ext v_16fold_tmp.16b, v0.16b, v_16fold_zero.16b, #4 + + dup d_16fold_from_h, v_16fold_from.d[1] + fmov d_16fold_p0_h, x_tmp1 + pmull v_16fold_from1.1q, v_16fold_from_h.1d, v_16fold_p0_h.1d + eor v_16fold_from2.16b, v_16fold_tmp.16b, v_16fold_from1.16b + + dup d_16fold_from2_h, v_16fold_from2.d[1] + fmov d_16fold_p0_l, x_tmp2 + pmull v6.1q, v_16fold_from2_h.1d, v_16fold_p0_l.1d + eor v_x0.16b, v0.16b, v6.16b + +/* carry less multiplication, part3 - after loop */ +/* crc16 barrett reduction function */ + +// input parameters: +// v_x0: v2 +// barrett reduction constant: br[0], br[1] + +d_br0 .req d3 +v_br0 .req v3 +d_br1 .req d5 +v_br1 .req v5 + + mov x_tmp1, 0x57f9 /* br[0] low */ + movk x_tmp1, 0xf65a, lsl 16 /* br[0] high */ + movk x_tmp1, 0x1, lsl 32 + fmov d_br0, x_tmp1 + + dup d1, v_x0.d[0] + dup d1, v1.d[0] + ext v1.16b, v1.16b, v7.16b, #4 + pmull v4.1q, v1.1d, v_br0.1d + + ext v1.16b, v4.16b, v7.16b, #4 + mov x_tmp1, 0x8bb70000 /* br[1] low */ + movk x_tmp1, 0x1, lsl 32 /* br[1] high */ + + fmov d_br1, x_tmp1 + pmull v_br1.1q, v1.1d, v_br1.1d + eor v_x0.16b, v_x0.16b, v_br1.16b + + umov x0, v_x0.d[0] + ubfx x0, x0, 16, 16 + b .crc_table_loop_pre + + .size crc16_t10dif_copy_pmull, .-crc16_t10dif_copy_pmull + + .section .rodata + + .align 4 +.shuffle_mask_lanchor = . + 0 + .type shuffle_mask, %object + .size shuffle_mask, 16 +shuffle_mask: + .byte 15, 14, 13, 12, 11, 10, 9, 8 + .byte 7, 6, 5, 4, 3, 2, 1, 0 + + .align 4 +.LANCHOR0 = . + 0 + .type crc16tab, %object + .size crc16tab, 512 +crc16tab: + .hword 0x0000, 0x8bb7, 0x9cd9, 0x176e, 0xb205, 0x39b2, 0x2edc, 0xa56b + .hword 0xEFBD, 0x640a, 0x7364, 0xf8d3, 0x5db8, 0xd60f, 0xc161, 0x4ad6 + .hword 0x54CD, 0xdf7a, 0xc814, 0x43a3, 0xe6c8, 0x6d7f, 0x7a11, 0xf1a6 + .hword 0xBB70, 0x30c7, 0x27a9, 0xac1e, 0x0975, 0x82c2, 0x95ac, 0x1e1b + .hword 0xA99A, 0x222d, 0x3543, 0xbef4, 0x1b9f, 0x9028, 0x8746, 0x0cf1 + .hword 0x4627, 0xcd90, 0xdafe, 0x5149, 0xf422, 0x7f95, 0x68fb, 0xe34c + .hword 0xFD57, 0x76e0, 0x618e, 0xea39, 0x4f52, 0xc4e5, 0xd38b, 0x583c + .hword 0x12EA, 0x995d, 0x8e33, 0x0584, 0xa0ef, 0x2b58, 0x3c36, 0xb781 + .hword 0xD883, 0x5334, 0x445a, 0xcfed, 0x6a86, 0xe131, 0xf65f, 0x7de8 + .hword 0x373E, 0xbc89, 0xabe7, 0x2050, 0x853b, 0x0e8c, 0x19e2, 0x9255 + .hword 0x8C4E, 0x07f9, 0x1097, 0x9b20, 0x3e4b, 0xb5fc, 0xa292, 0x2925 + .hword 0x63F3, 0xe844, 0xff2a, 0x749d, 0xd1f6, 0x5a41, 0x4d2f, 0xc698 + .hword 0x7119, 0xfaae, 0xedc0, 0x6677, 0xc31c, 0x48ab, 0x5fc5, 0xd472 + .hword 0x9EA4, 0x1513, 0x027d, 0x89ca, 0x2ca1, 0xa716, 0xb078, 0x3bcf + .hword 0x25D4, 0xae63, 0xb90d, 0x32ba, 0x97d1, 0x1c66, 0x0b08, 0x80bf + .hword 0xCA69, 0x41de, 0x56b0, 0xdd07, 0x786c, 0xf3db, 0xe4b5, 0x6f02 + .hword 0x3AB1, 0xb106, 0xa668, 0x2ddf, 0x88b4, 0x0303, 0x146d, 0x9fda + .hword 0xD50C, 0x5ebb, 0x49d5, 0xc262, 0x6709, 0xecbe, 0xfbd0, 0x7067 + .hword 0x6E7C, 0xe5cb, 0xf2a5, 0x7912, 0xdc79, 0x57ce, 0x40a0, 0xcb17 + .hword 0x81C1, 0x0a76, 0x1d18, 0x96af, 0x33c4, 0xb873, 0xaf1d, 0x24aa + .hword 0x932B, 0x189c, 0x0ff2, 0x8445, 0x212e, 0xaa99, 0xbdf7, 0x3640 + .hword 0x7C96, 0xf721, 0xe04f, 0x6bf8, 0xce93, 0x4524, 0x524a, 0xd9fd + .hword 0xC7E6, 0x4c51, 0x5b3f, 0xd088, 0x75e3, 0xfe54, 0xe93a, 0x628d + .hword 0x285B, 0xa3ec, 0xb482, 0x3f35, 0x9a5e, 0x11e9, 0x0687, 0x8d30 + .hword 0xE232, 0x6985, 0x7eeb, 0xf55c, 0x5037, 0xdb80, 0xccee, 0x4759 + .hword 0x0D8F, 0x8638, 0x9156, 0x1ae1, 0xbf8a, 0x343d, 0x2353, 0xa8e4 + .hword 0xB6FF, 0x3d48, 0x2a26, 0xa191, 0x04fa, 0x8f4d, 0x9823, 0x1394 + .hword 0x5942, 0xd2f5, 0xc59b, 0x4e2c, 0xeb47, 0x60f0, 0x779e, 0xfc29 + .hword 0x4BA8, 0xc01f, 0xd771, 0x5cc6, 0xf9ad, 0x721a, 0x6574, 0xeec3 + .hword 0xA415, 0x2fa2, 0x38cc, 0xb37b, 0x1610, 0x9da7, 0x8ac9, 0x017e + .hword 0x1F65, 0x94d2, 0x83bc, 0x080b, 0xad60, 0x26d7, 0x31b9, 0xba0e + .hword 0xF0D8, 0x7b6f, 0x6c01, 0xe7b6, 0x42dd, 0xc96a, 0xde04, 0x55b3 diff --git a/src/isa-l/crc/aarch64/crc16_t10dif_pmull.S b/src/isa-l/crc/aarch64/crc16_t10dif_pmull.S new file mode 100644 index 000000000..08f1a35ad --- /dev/null +++ b/src/isa-l/crc/aarch64/crc16_t10dif_pmull.S @@ -0,0 +1,404 @@ +######################################################################## +# Copyright(c) 2019 Arm Corporation All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Arm Corporation nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +######################################################################### + + .arch armv8-a+crc+crypto + .text + .align 3 + .global crc16_t10dif_pmull + .type crc16_t10dif_pmull, %function + +/* uint16_t crc16_t10dif_pmull(uint16_t seed, uint8_t *buf, uint64_t len) */ + +/* arguments */ +w_seed .req w0 +x_buf .req x1 +x_len .req x2 +w_len .req w2 + +/* returns */ +w_ret .req w0 + +/* these as global temporary registers */ +w_tmp .req w5 +x_tmp .req x5 +x_tmp1 .req x6 +x_tmp2 .req x7 + +d_tmp1 .req d0 +d_tmp2 .req d1 +q_tmp1 .req q0 +q_tmp2 .req q1 +v_tmp1 .req v0 +v_tmp2 .req v1 + +/* local variables */ +w_counter .req w3 +w_crc .req w0 +x_crc .req x0 +x_counter .req x3 +x_crc16tab .req x4 +x_buf_saved .req x0 + +crc16_t10dif_pmull: + cmp x_len, 1023 + sub sp, sp, #16 + uxth w_seed, w_seed + bhi .crc_fold + + mov x_tmp, 0 + mov w_counter, 0 + +.crc_table_loop_pre: + cmp x_len, x_tmp + bls .end + + sxtw x_counter, w_counter + adrp x_crc16tab, .LANCHOR0 + sub x_buf, x_buf, x_counter + add x_crc16tab, x_crc16tab, :lo12:.LANCHOR0 + + .align 2 +.crc_table_loop: + ldrb w_tmp, [x_buf, x_counter] + add x_counter, x_counter, 1 + cmp x_len, x_counter + eor w_tmp, w_tmp, w_crc, lsr 8 + ldrh w_tmp, [x_crc16tab, w_tmp, sxtw 1] + eor w_crc, w_tmp, w_crc, lsl 8 + uxth w_crc, w_crc + bhi .crc_table_loop + +.end: + add sp, sp, 16 + ret + +/* carry less multiplication, part1 - before loop */ +q_x0 .req q2 +q_x1 .req q3 +q_x2 .req q4 +q_x3 .req q5 + +v_x0 .req v2 +v_x1 .req v3 +v_x2 .req v4 +v_x3 .req v5 + +d_x0 .req d2 +d_x1 .req d3 +d_x2 .req d4 +d_x3 .req d5 + +// the following registers only used this part1 +d_tmp3 .req d16 +v_tmp3 .req v16 + + .align 3 +.crc_fold: + fmov d_tmp1, x_crc + fmov d_tmp2, xzr + dup d_tmp3, v_tmp2.d[0] + shl d_tmp1, d_tmp1, 48 + ins v_tmp3.d[1], v_tmp1.d[0] + + and x_counter, x_len, -64 + sub x_counter, x_counter, #64 + cmp x_counter, 63 + add x_buf_saved, x_buf, 64 + + ldr q_x0, [x_buf] + ldr q_x1, [x_buf, 16] + ldr q_x2, [x_buf, 32] + ldr q_x3, [x_buf, 48] + + adrp x_tmp, .shuffle_mask_lanchor + ldr q7, [x_tmp, :lo12:.shuffle_mask_lanchor] + + tbl v_tmp1.16b, {v_x0.16b}, v7.16b + eor v_x0.16b, v_tmp3.16b, v_tmp1.16b + + tbl v_x1.16b, {v_x1.16b}, v7.16b + tbl v_x2.16b, {v_x2.16b}, v7.16b + tbl v_x3.16b, {v_x3.16b}, v7.16b + bls .crc_fold_loop_end + +/* carry less multiplication, part2 - loop */ +q_y0 .req q28 +q_y1 .req q29 +q_y2 .req q30 +q_y3 .req q31 + +v_y0 .req v28 +v_y1 .req v29 +v_y2 .req v30 +v_y3 .req v31 + +d_x0_h .req d24 +d_x0_l .req d2 +d_x1_h .req d25 +d_x1_l .req d3 +d_x2_h .req d26 +d_x2_l .req d4 +d_x3_h .req d27 +d_x3_l .req d5 + +v_x0_h .req v24 +v_x0_l .req v2 +v_x1_h .req v25 +v_x1_l .req v3 +v_x2_h .req v26 +v_x2_l .req v4 +v_x3_h .req v27 +v_x3_l .req v5 + +v_tmp1_x0 .req v24 +v_tmp1_x1 .req v25 +v_tmp1_x2 .req v26 +v_tmp1_x3 .req v27 + +d_p4_h .req d19 +v_p4_h .req v19 +d_p4_l .req d17 +v_p4_l .req v17 + + mov x_tmp, 0x371d0000 /* p4 [1] */ + fmov d_p4_h, x_tmp + mov x_tmp, 0x87e70000 /* p4 [0] */ + fmov d_p4_l, x_tmp + + .align 2 +.crc_fold_loop: + add x_buf_saved, x_buf_saved, 64 + sub x_counter, x_counter, #64 + cmp x_counter, 63 + + dup d_x0_h, v_x0.d[1] + dup d_x1_h, v_x1.d[1] + dup d_x2_h, v_x2.d[1] + dup d_x3_h, v_x3.d[1] + + dup d_x0_l, v_x0.d[0] + dup d_x1_l, v_x1.d[0] + dup d_x2_l, v_x2.d[0] + dup d_x3_l, v_x3.d[0] + + ldr q_y0, [x_buf_saved, -64] + ldr q_y1, [x_buf_saved, -48] + ldr q_y2, [x_buf_saved, -32] + ldr q_y3, [x_buf_saved, -16] + + pmull v_x0_h.1q, v_x0_h.1d, v_p4_h.1d + pmull v_x0_l.1q, v_x0_l.1d, v_p4_l.1d + pmull v_x1_h.1q, v_x1_h.1d, v_p4_h.1d + pmull v_x1_l.1q, v_x1_l.1d, v_p4_l.1d + pmull v_x2_h.1q, v_x2_h.1d, v_p4_h.1d + pmull v_x2_l.1q, v_x2_l.1d, v_p4_l.1d + pmull v_x3_h.1q, v_x3_h.1d, v_p4_h.1d + pmull v_x3_l.1q, v_x3_l.1d, v_p4_l.1d + + tbl v_y0.16b, {v_y0.16b}, v7.16b + tbl v_y1.16b, {v_y1.16b}, v7.16b + tbl v_y2.16b, {v_y2.16b}, v7.16b + tbl v_y3.16b, {v_y3.16b}, v7.16b + + eor v_tmp1_x0.16b, v_x0_h.16b, v_x0_l.16b + eor v_tmp1_x1.16b, v_x1_h.16b, v_x1_l.16b + eor v_tmp1_x2.16b, v_x2_h.16b, v_x2_l.16b + eor v_tmp1_x3.16b, v_x3_h.16b, v_x3_l.16b + + eor v_x0.16b, v_tmp1_x0.16b, v_y0.16b + eor v_x1.16b, v_tmp1_x1.16b, v_y1.16b + eor v_x2.16b, v_tmp1_x2.16b, v_y2.16b + eor v_x3.16b, v_tmp1_x3.16b, v_y3.16b + + bhi .crc_fold_loop + +/* carry less multiplication, part3 - after loop */ +/* folding 512bit ---> 128bit */ + +// input parameters: +// v_x0 => v2 +// v_x1 => v3 +// v_x2 => v4 +// v_x3 => v5 + +// v0, v1, v6, v30, are tmp registers + +.crc_fold_loop_end: + mov x_tmp, 0x4c1a0000 /* p1 [1] */ + fmov d0, x_tmp + mov x_tmp, 0xfb0b0000 /* p1 [0] */ + fmov d1, x_tmp + + and w_counter, w_len, -64 + sxtw x_tmp, w_counter + add x_buf, x_buf, x_tmp + + dup d6, v_x0.d[1] + dup d30, v_x0.d[0] + pmull v6.1q, v6.1d, v0.1d + pmull v30.1q, v30.1d, v1.1d + eor v6.16b, v6.16b, v30.16b + eor v_x1.16b, v6.16b, v_x1.16b + + dup d6, v_x1.d[1] + dup d30, v_x1.d[0] + pmull v6.1q, v6.1d, v0.1d + pmull v16.1q, v30.1d, v1.1d + eor v6.16b, v6.16b, v16.16b + eor v_x2.16b, v6.16b, v_x2.16b + + dup d_x0, v_x2.d[1] + dup d30, v_x2.d[0] + pmull v0.1q, v_x0.1d, v0.1d + pmull v_x0.1q, v30.1d, v1.1d + eor v1.16b, v0.16b, v_x0.16b + eor v_x0.16b, v1.16b, v_x3.16b + +/* carry less multiplication, part3 - after loop */ +/* crc16 fold function */ +d_16fold_p0_h .req d18 +v_16fold_p0_h .req v18 + +d_16fold_p0_l .req d4 +v_16fold_p0_l .req v4 + +v_16fold_from .req v_x0 +d_16fold_from_h .req d3 +v_16fold_from_h .req v3 + +v_16fold_zero .req v7 + +v_16fold_from1 .req v16 + +v_16fold_from2 .req v0 +d_16fold_from2_h .req d6 +v_16fold_from2_h .req v6 + +v_16fold_tmp .req v0 + + movi v_16fold_zero.4s, 0 + mov x_tmp1, 0x2d560000 /* p0 [1] */ + mov x_tmp2, 0x13680000 /* p0 [0] */ + + ext v_16fold_tmp.16b, v_16fold_zero.16b, v_16fold_from.16b, #8 + ext v_16fold_tmp.16b, v0.16b, v_16fold_zero.16b, #4 + + dup d_16fold_from_h, v_16fold_from.d[1] + fmov d_16fold_p0_h, x_tmp1 + pmull v_16fold_from1.1q, v_16fold_from_h.1d, v_16fold_p0_h.1d + eor v_16fold_from2.16b, v_16fold_tmp.16b, v_16fold_from1.16b + + dup d_16fold_from2_h, v_16fold_from2.d[1] + fmov d_16fold_p0_l, x_tmp2 + pmull v6.1q, v_16fold_from2_h.1d, v_16fold_p0_l.1d + eor v_x0.16b, v0.16b, v6.16b + +/* carry less multiplication, part3 - after loop */ +/* crc16 barrett reduction function */ + +// input parameters: +// v_x0: v2 +// barrett reduction constant: br[0], br[1] + +d_br0 .req d3 +v_br0 .req v3 +d_br1 .req d5 +v_br1 .req v5 + + mov x_tmp1, 0x57f9 /* br[0] low */ + movk x_tmp1, 0xf65a, lsl 16 /* br[0] high */ + movk x_tmp1, 0x1, lsl 32 + fmov d_br0, x_tmp1 + + dup d1, v_x0.d[0] + dup d1, v1.d[0] + ext v1.16b, v1.16b, v7.16b, #4 + pmull v4.1q, v1.1d, v_br0.1d + + ext v1.16b, v4.16b, v7.16b, #4 + mov x_tmp1, 0x8bb70000 /* br[1] low */ + movk x_tmp1, 0x1, lsl 32 /* br[1] high */ + + fmov d_br1, x_tmp1 + pmull v_br1.1q, v1.1d, v_br1.1d + eor v_x0.16b, v_x0.16b, v_br1.16b + + umov x0, v_x0.d[0] + ubfx x0, x0, 16, 16 + b .crc_table_loop_pre + + .size crc16_t10dif_pmull, .-crc16_t10dif_pmull + + .section .rodata + + .align 4 +.shuffle_mask_lanchor = . + 0 + .type shuffle_mask, %object + .size shuffle_mask, 16 +shuffle_mask: + .byte 15, 14, 13, 12, 11, 10, 9, 8 + .byte 7, 6, 5, 4, 3, 2, 1, 0 + + .align 4 +.LANCHOR0 = . + 0 + .type crc16tab, %object + .size crc16tab, 512 +crc16tab: + .hword 0x0000, 0x8bb7, 0x9cd9, 0x176e, 0xb205, 0x39b2, 0x2edc, 0xa56b + .hword 0xEFBD, 0x640a, 0x7364, 0xf8d3, 0x5db8, 0xd60f, 0xc161, 0x4ad6 + .hword 0x54CD, 0xdf7a, 0xc814, 0x43a3, 0xe6c8, 0x6d7f, 0x7a11, 0xf1a6 + .hword 0xBB70, 0x30c7, 0x27a9, 0xac1e, 0x0975, 0x82c2, 0x95ac, 0x1e1b + .hword 0xA99A, 0x222d, 0x3543, 0xbef4, 0x1b9f, 0x9028, 0x8746, 0x0cf1 + .hword 0x4627, 0xcd90, 0xdafe, 0x5149, 0xf422, 0x7f95, 0x68fb, 0xe34c + .hword 0xFD57, 0x76e0, 0x618e, 0xea39, 0x4f52, 0xc4e5, 0xd38b, 0x583c + .hword 0x12EA, 0x995d, 0x8e33, 0x0584, 0xa0ef, 0x2b58, 0x3c36, 0xb781 + .hword 0xD883, 0x5334, 0x445a, 0xcfed, 0x6a86, 0xe131, 0xf65f, 0x7de8 + .hword 0x373E, 0xbc89, 0xabe7, 0x2050, 0x853b, 0x0e8c, 0x19e2, 0x9255 + .hword 0x8C4E, 0x07f9, 0x1097, 0x9b20, 0x3e4b, 0xb5fc, 0xa292, 0x2925 + .hword 0x63F3, 0xe844, 0xff2a, 0x749d, 0xd1f6, 0x5a41, 0x4d2f, 0xc698 + .hword 0x7119, 0xfaae, 0xedc0, 0x6677, 0xc31c, 0x48ab, 0x5fc5, 0xd472 + .hword 0x9EA4, 0x1513, 0x027d, 0x89ca, 0x2ca1, 0xa716, 0xb078, 0x3bcf + .hword 0x25D4, 0xae63, 0xb90d, 0x32ba, 0x97d1, 0x1c66, 0x0b08, 0x80bf + .hword 0xCA69, 0x41de, 0x56b0, 0xdd07, 0x786c, 0xf3db, 0xe4b5, 0x6f02 + .hword 0x3AB1, 0xb106, 0xa668, 0x2ddf, 0x88b4, 0x0303, 0x146d, 0x9fda + .hword 0xD50C, 0x5ebb, 0x49d5, 0xc262, 0x6709, 0xecbe, 0xfbd0, 0x7067 + .hword 0x6E7C, 0xe5cb, 0xf2a5, 0x7912, 0xdc79, 0x57ce, 0x40a0, 0xcb17 + .hword 0x81C1, 0x0a76, 0x1d18, 0x96af, 0x33c4, 0xb873, 0xaf1d, 0x24aa + .hword 0x932B, 0x189c, 0x0ff2, 0x8445, 0x212e, 0xaa99, 0xbdf7, 0x3640 + .hword 0x7C96, 0xf721, 0xe04f, 0x6bf8, 0xce93, 0x4524, 0x524a, 0xd9fd + .hword 0xC7E6, 0x4c51, 0x5b3f, 0xd088, 0x75e3, 0xfe54, 0xe93a, 0x628d + .hword 0x285B, 0xa3ec, 0xb482, 0x3f35, 0x9a5e, 0x11e9, 0x0687, 0x8d30 + .hword 0xE232, 0x6985, 0x7eeb, 0xf55c, 0x5037, 0xdb80, 0xccee, 0x4759 + .hword 0x0D8F, 0x8638, 0x9156, 0x1ae1, 0xbf8a, 0x343d, 0x2353, 0xa8e4 + .hword 0xB6FF, 0x3d48, 0x2a26, 0xa191, 0x04fa, 0x8f4d, 0x9823, 0x1394 + .hword 0x5942, 0xd2f5, 0xc59b, 0x4e2c, 0xeb47, 0x60f0, 0x779e, 0xfc29 + .hword 0x4BA8, 0xc01f, 0xd771, 0x5cc6, 0xf9ad, 0x721a, 0x6574, 0xeec3 + .hword 0xA415, 0x2fa2, 0x38cc, 0xb37b, 0x1610, 0x9da7, 0x8ac9, 0x017e + .hword 0x1F65, 0x94d2, 0x83bc, 0x080b, 0xad60, 0x26d7, 0x31b9, 0xba0e + .hword 0xF0D8, 0x7b6f, 0x6c01, 0xe7b6, 0x42dd, 0xc96a, 0xde04, 0x55b3 diff --git a/src/isa-l/crc/aarch64/crc32_aarch64_common.h b/src/isa-l/crc/aarch64/crc32_aarch64_common.h new file mode 100644 index 000000000..a2ef22aea --- /dev/null +++ b/src/isa-l/crc/aarch64/crc32_aarch64_common.h @@ -0,0 +1,321 @@ +/********************************************************************** + Copyright(c) 2020 Arm Corporation All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Arm Corporation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +**********************************************************************/ + + + + +.macro crc32_hw_common poly_type + +.ifc \poly_type,crc32 + mvn wCRC,wCRC +.endif + cbz LEN, .zero_length_ret + tbz BUF, 0, .align_short + ldrb wdata,[BUF],1 + sub LEN,LEN,1 + crc32_u8 wCRC,wCRC,wdata +.align_short: + tst BUF,2 + ccmp LEN,1,0,ne + bhi .align_short_2 + tst BUF,4 + ccmp LEN,3,0,ne + bhi .align_word + +.align_finish: + + cmp LEN, 63 + bls .loop_16B +.loop_64B: + ldp data0, data1, [BUF],#16 + prfm pldl2keep,[BUF,2048] + sub LEN,LEN,#64 + ldp data2, data3, [BUF],#16 + prfm pldl1keep,[BUF,256] + cmp LEN,#64 + crc32_u64 wCRC, wCRC, data0 + crc32_u64 wCRC, wCRC, data1 + ldp data0, data1, [BUF],#16 + crc32_u64 wCRC, wCRC, data2 + crc32_u64 wCRC, wCRC, data3 + ldp data2, data3, [BUF],#16 + crc32_u64 wCRC, wCRC, data0 + crc32_u64 wCRC, wCRC, data1 + crc32_u64 wCRC, wCRC, data2 + crc32_u64 wCRC, wCRC, data3 + bge .loop_64B + +.loop_16B: + cmp LEN, 15 + bls .less_16B + ldp data0, data1, [BUF],#16 + sub LEN,LEN,#16 + cmp LEN,15 + crc32_u64 wCRC, wCRC, data0 + crc32_u64 wCRC, wCRC, data1 + bls .less_16B + ldp data0, data1, [BUF],#16 + sub LEN,LEN,#16 + cmp LEN,15 + crc32_u64 wCRC, wCRC, data0 + crc32_u64 wCRC, wCRC, data1 + bls .less_16B + ldp data0, data1, [BUF],#16 + sub LEN,LEN,#16 //MUST less than 16B + crc32_u64 wCRC, wCRC, data0 + crc32_u64 wCRC, wCRC, data1 +.less_16B: + cmp LEN, 7 + bls .less_8B + ldr data0, [BUF], 8 + sub LEN, LEN, #8 + crc32_u64 wCRC, wCRC, data0 +.less_8B: + cmp LEN, 3 + bls .less_4B + ldr wdata, [BUF], 4 + sub LEN, LEN, #4 + crc32_u32 wCRC, wCRC, wdata +.less_4B: + cmp LEN, 1 + bls .less_2B + ldrh wdata, [BUF], 2 + sub LEN, LEN, #2 + crc32_u16 wCRC, wCRC, wdata +.less_2B: + cbz LEN, .zero_length_ret + ldrb wdata, [BUF] + crc32_u8 wCRC, wCRC, wdata +.zero_length_ret: +.ifc \poly_type,crc32 + mvn w0, wCRC +.else + mov w0, wCRC +.endif + ret +.align_short_2: + ldrh wdata, [BUF], 2 + sub LEN, LEN, 2 + tst BUF, 4 + crc32_u16 wCRC, wCRC, wdata + ccmp LEN, 3, 0, ne + bls .align_finish +.align_word: + ldr wdata, [BUF], 4 + sub LEN, LEN, #4 + crc32_u32 wCRC, wCRC, wdata + b .align_finish +.endm + +.macro crc32_3crc_fold poly_type +.ifc \poly_type,crc32 + mvn wCRC,wCRC +.endif + cbz LEN, .zero_length_ret + tbz BUF, 0, .align_short + ldrb wdata,[BUF],1 + sub LEN,LEN,1 + crc32_u8 wCRC,wCRC,wdata +.align_short: + tst BUF,2 + ccmp LEN,1,0,ne + bhi .align_short_2 + tst BUF,4 + ccmp LEN,3,0,ne + bhi .align_word + +.align_finish: + cmp LEN,1023 + adr const_adr, .Lconstants + bls 1f + ldp dconst0,dconst1,[const_adr] +2: + ldr crc0_data0,[ptr_crc0],8 + prfm pldl2keep,[ptr_crc0,3*1024-8] + mov crc1,0 + mov crc2,0 + add ptr_crc1,ptr_crc0,336 + add ptr_crc2,ptr_crc0,336*2 + crc32_u64 crc0,crc0,crc0_data0 + .set offset,0 + .set ptr_offset,8 + .rept 5 + ldp crc0_data0,crc0_data1,[ptr_crc0],16 + ldp crc1_data0,crc1_data1,[ptr_crc1],16 + .set offset,offset+64 + .set ptr_offset,ptr_offset+16 + prfm pldl2keep,[ptr_crc0,3*1024-ptr_offset+offset] + crc32_u64 crc0,crc0,crc0_data0 + crc32_u64 crc0,crc0,crc0_data1 + ldp crc2_data0,crc2_data1,[ptr_crc2],16 + crc32_u64 crc1,crc1,crc1_data0 + crc32_u64 crc1,crc1,crc1_data1 + crc32_u64 crc2,crc2,crc2_data0 + crc32_u64 crc2,crc2,crc2_data1 + .endr + .set l1_offset,0 + .rept 10 + ldp crc0_data0,crc0_data1,[ptr_crc0],16 + ldp crc1_data0,crc1_data1,[ptr_crc1],16 + .set offset,offset+64 + .set ptr_offset,ptr_offset+16 + prfm pldl2keep,[ptr_crc0,3*1024-ptr_offset+offset] + prfm pldl1keep,[ptr_crc0,2*1024-ptr_offset+l1_offset] + .set l1_offset,l1_offset+64 + crc32_u64 crc0,crc0,crc0_data0 + crc32_u64 crc0,crc0,crc0_data1 + ldp crc2_data0,crc2_data1,[ptr_crc2],16 + crc32_u64 crc1,crc1,crc1_data0 + crc32_u64 crc1,crc1,crc1_data1 + crc32_u64 crc2,crc2,crc2_data0 + crc32_u64 crc2,crc2,crc2_data1 + .endr + + .rept 6 + ldp crc0_data0,crc0_data1,[ptr_crc0],16 + ldp crc1_data0,crc1_data1,[ptr_crc1],16 + .set ptr_offset,ptr_offset+16 + prfm pldl1keep,[ptr_crc0,2*1024-ptr_offset+l1_offset] + .set l1_offset,l1_offset+64 + crc32_u64 crc0,crc0,crc0_data0 + crc32_u64 crc0,crc0,crc0_data1 + ldp crc2_data0,crc2_data1,[ptr_crc2],16 + crc32_u64 crc1,crc1,crc1_data0 + crc32_u64 crc1,crc1,crc1_data1 + crc32_u64 crc2,crc2,crc2_data0 + crc32_u64 crc2,crc2,crc2_data1 + .endr + ldr crc2_data0,[ptr_crc2] + fmov dtmp0,xcrc0 + fmov dtmp1,xcrc1 + crc32_u64 crc2,crc2,crc2_data0 + add ptr_crc0,ptr_crc0,1024-(336+8) + pmull vtmp0.1q,vtmp0.1d,vconst0.1d + sub LEN,LEN,1024 + pmull vtmp1.1q,vtmp1.1d,vconst1.1d + cmp LEN,1024 + fmov xcrc0,dtmp0 + fmov xcrc1,dtmp1 + crc32_u64 crc0,wzr,xcrc0 + crc32_u64 crc1,wzr,xcrc1 + + eor crc0,crc0,crc2 + eor crc0,crc0,crc1 + + bhs 2b +1: + cmp LEN, 63 + bls .loop_16B +.loop_64B: + ldp data0, data1, [BUF],#16 + sub LEN,LEN,#64 + ldp data2, data3, [BUF],#16 + cmp LEN,#64 + crc32_u64 wCRC, wCRC, data0 + crc32_u64 wCRC, wCRC, data1 + ldp data0, data1, [BUF],#16 + crc32_u64 wCRC, wCRC, data2 + crc32_u64 wCRC, wCRC, data3 + ldp data2, data3, [BUF],#16 + crc32_u64 wCRC, wCRC, data0 + crc32_u64 wCRC, wCRC, data1 + crc32_u64 wCRC, wCRC, data2 + crc32_u64 wCRC, wCRC, data3 + bge .loop_64B + +.loop_16B: + cmp LEN, 15 + bls .less_16B + ldp data0, data1, [BUF],#16 + sub LEN,LEN,#16 + cmp LEN,15 + crc32_u64 wCRC, wCRC, data0 + crc32_u64 wCRC, wCRC, data1 + bls .less_16B + ldp data0, data1, [BUF],#16 + sub LEN,LEN,#16 + cmp LEN,15 + crc32_u64 wCRC, wCRC, data0 + crc32_u64 wCRC, wCRC, data1 + bls .less_16B + ldp data0, data1, [BUF],#16 + sub LEN,LEN,#16 //MUST less than 16B + crc32_u64 wCRC, wCRC, data0 + crc32_u64 wCRC, wCRC, data1 +.less_16B: + cmp LEN, 7 + bls .less_8B + ldr data0, [BUF], 8 + sub LEN, LEN, #8 + crc32_u64 wCRC, wCRC, data0 +.less_8B: + cmp LEN, 3 + bls .less_4B + ldr wdata, [BUF], 4 + sub LEN, LEN, #4 + crc32_u32 wCRC, wCRC, wdata +.less_4B: + cmp LEN, 1 + bls .less_2B + ldrh wdata, [BUF], 2 + sub LEN, LEN, #2 + crc32_u16 wCRC, wCRC, wdata +.less_2B: + cbz LEN, .zero_length_ret + ldrb wdata, [BUF] + crc32_u8 wCRC, wCRC, wdata +.zero_length_ret: +.ifc \poly_type,crc32 + mvn w0, wCRC +.else + mov w0, wCRC +.endif + ret +.align_short_2: + ldrh wdata, [BUF], 2 + sub LEN, LEN, 2 + tst BUF, 4 + crc32_u16 wCRC, wCRC, wdata + ccmp LEN, 3, 0, ne + bls .align_finish +.align_word: + ldr wdata, [BUF], 4 + sub LEN, LEN, #4 + crc32_u32 wCRC, wCRC, wdata + b .align_finish +.Lconstants: +.ifc \poly_type,crc32 + .quad 0xb486819b + .quad 0x76278617 +.else + .quad 0xe417f38a + .quad 0x8f158014 +.endif + +.endm diff --git a/src/isa-l/crc/aarch64/crc32_common_crc_ext_cortex_a72.S b/src/isa-l/crc/aarch64/crc32_common_crc_ext_cortex_a72.S new file mode 100644 index 000000000..7c9ca35ad --- /dev/null +++ b/src/isa-l/crc/aarch64/crc32_common_crc_ext_cortex_a72.S @@ -0,0 +1,135 @@ +/********************************************************************** + Copyright(c) 2020 Arm Corporation All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Arm Corporation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +**********************************************************************/ + + + + +.macro crc32_hw_common poly_type + cbz LEN, .zero_length_ret +.ifc \poly_type,crc32 + mvn wCRC,wCRC +.endif + tbz BUF, 0, .align_short + ldrb wdata,[BUF],1 + sub LEN,LEN,1 + crc32_u8 wCRC,wCRC,wdata +.align_short: + tst BUF,2 + ccmp LEN,1,0,ne + bhi .align_short_2 + tst BUF,4 + ccmp LEN,3,0,ne + bhi .align_word + +.align_finish: + + cmp LEN, 63 + bls .loop_16B +.loop_64B: + ldp data0, data1, [BUF],#16 + sub LEN,LEN,#64 + ldp data2, data3, [BUF],#16 + cmp LEN,#64 + crc32_u64 wCRC, wCRC, data0 + crc32_u64 wCRC, wCRC, data1 + ldp data0, data1, [BUF],#16 + crc32_u64 wCRC, wCRC, data2 + crc32_u64 wCRC, wCRC, data3 + ldp data2, data3, [BUF],#16 + crc32_u64 wCRC, wCRC, data0 + crc32_u64 wCRC, wCRC, data1 + crc32_u64 wCRC, wCRC, data2 + crc32_u64 wCRC, wCRC, data3 + bge .loop_64B + +.loop_16B: + cmp LEN, 15 + bls .less_16B + ldp data0, data1, [BUF],#16 + sub LEN,LEN,#16 + cmp LEN,15 + crc32_u64 wCRC, wCRC, data0 + crc32_u64 wCRC, wCRC, data1 + bls .less_16B + ldp data0, data1, [BUF],#16 + sub LEN,LEN,#16 + cmp LEN,15 + crc32_u64 wCRC, wCRC, data0 + crc32_u64 wCRC, wCRC, data1 + bls .less_16B + ldp data0, data1, [BUF],#16 + sub LEN,LEN,#16 //MUST less than 16B + crc32_u64 wCRC, wCRC, data0 + crc32_u64 wCRC, wCRC, data1 +.less_16B: + cmp LEN, 7 + bls .less_8B + ldr data0, [BUF], 8 + sub LEN, LEN, #8 + crc32_u64 wCRC, wCRC, data0 +.less_8B: + cmp LEN, 3 + bls .less_4B + ldr wdata, [BUF], 4 + sub LEN, LEN, #4 + crc32_u32 wCRC, wCRC, wdata +.less_4B: + cmp LEN, 1 + bls .less_2B + ldrh wdata, [BUF], 2 + sub LEN, LEN, #2 + crc32_u16 wCRC, wCRC, wdata +.less_2B: + cbz LEN, .finish_exit + ldrb wdata, [BUF] + crc32_u8 wCRC, wCRC, wdata +.finish_exit: +.ifc \poly_type,crc32 + mvn w0, wCRC +.else + mov w0, wCRC +.endif + ret +.zero_length_ret: + mov w0, wCRC + ret +.align_short_2: + ldrh wdata, [BUF], 2 + sub LEN, LEN, 2 + tst BUF, 4 + crc32_u16 wCRC, wCRC, wdata + ccmp LEN, 3, 0, ne + bls .align_finish +.align_word: + ldr wdata, [BUF], 4 + sub LEN, LEN, #4 + crc32_u32 wCRC, wCRC, wdata + b .align_finish + +.endm diff --git a/src/isa-l/crc/aarch64/crc32_common_mix_neoverse_n1.S b/src/isa-l/crc/aarch64/crc32_common_mix_neoverse_n1.S new file mode 100644 index 000000000..4911a30b8 --- /dev/null +++ b/src/isa-l/crc/aarch64/crc32_common_mix_neoverse_n1.S @@ -0,0 +1,432 @@ +/********************************************************************** + Copyright(c) 2020 Arm Corporation All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Arm Corporation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +**********************************************************************/ + + +.macro declare_var_vector_reg name:req,reg:req + \name\()_q .req q\reg + \name\()_v .req v\reg + \name\()_s .req s\reg + \name\()_d .req d\reg +.endm + declare_var_vector_reg k1k2,20 + declare_var_vector_reg k3k4,21 + declare_var_vector_reg poly,22 + declare_var_vector_reg k5k0,23 + declare_var_vector_reg mask,24 + declare_var_vector_reg fold_poly,25 + + declare_var_vector_reg tmp0,0 + declare_var_vector_reg tmp1,1 + declare_var_vector_reg tmp2,2 + declare_var_vector_reg tmp3,3 + declare_var_vector_reg tmp4,4 + declare_var_vector_reg tmp5,5 + declare_var_vector_reg tmp6,6 + declare_var_vector_reg tmp7,7 + declare_var_vector_reg pmull_data0,16 + declare_var_vector_reg pmull_data1,17 + declare_var_vector_reg pmull_data2,18 + declare_var_vector_reg pmull_data3,19 + + vzr .req v26 + + const_addr .req x3 + crc_blk_ptr .req x4 + pmull_blk_ptr .req x5 + crc_data0 .req x6 + crc_data1 .req x7 + crc_data2 .req x9 + crc_data3 .req x10 + wPmull .req w11 + xPmull .req x11 + + data0 .req x4 + data1 .req x5 + data2 .req x6 + data3 .req x7 + wdata .req w4 + +.macro pmull_fold + + pmull2 tmp4_v.1q, tmp0_v.2d, k1k2_v.2d + pmull2 tmp5_v.1q, tmp1_v.2d, k1k2_v.2d + pmull2 tmp6_v.1q, tmp2_v.2d, k1k2_v.2d + pmull2 tmp7_v.1q, tmp3_v.2d, k1k2_v.2d + + pmull tmp0_v.1q, tmp0_v.1d, k1k2_v.1d + pmull tmp1_v.1q, tmp1_v.1d, k1k2_v.1d + pmull tmp2_v.1q, tmp2_v.1d, k1k2_v.1d + pmull tmp3_v.1q, tmp3_v.1d, k1k2_v.1d + ld1 {pmull_data0_v.16b-pmull_data3_v.16b},[pmull_blk_ptr],#64 + crc32_u64 wCRC,wCRC,crc_data0 + crc32_u64 wCRC,wCRC,crc_data1 + ldp crc_data0,crc_data1,[crc_blk_ptr],16 + crc32_u64 wCRC,wCRC,crc_data2 + crc32_u64 wCRC,wCRC,crc_data3 + ldp crc_data2,crc_data3,[crc_blk_ptr],16 + + eor tmp0_v.16b, tmp0_v.16b, tmp4_v.16b + eor tmp1_v.16b, tmp1_v.16b, tmp5_v.16b + eor tmp2_v.16b, tmp2_v.16b, tmp6_v.16b + eor tmp3_v.16b, tmp3_v.16b, tmp7_v.16b + + crc32_u64 wCRC,wCRC,crc_data0 + crc32_u64 wCRC,wCRC,crc_data1 + ldp crc_data0,crc_data1,[crc_blk_ptr],16 + crc32_u64 wCRC,wCRC,crc_data2 + crc32_u64 wCRC,wCRC,crc_data3 + ldp crc_data2,crc_data3,[crc_blk_ptr],16 + eor tmp0_v.16b, tmp0_v.16b, v16.16b + eor tmp1_v.16b, tmp1_v.16b, v17.16b + eor tmp2_v.16b, tmp2_v.16b, v18.16b + eor tmp3_v.16b, tmp3_v.16b, v19.16b + crc32_u64 wCRC,wCRC,crc_data0 + crc32_u64 wCRC,wCRC,crc_data1 + ldp crc_data0,crc_data1,[crc_blk_ptr],16 + crc32_u64 wCRC,wCRC,crc_data2 + crc32_u64 wCRC,wCRC,crc_data3 + ldp crc_data2,crc_data3,[crc_blk_ptr],16 +.endm + + + +.macro crc32_common_mix poly_type + .set MIX_BLK_SIZE,2048 + +.ifc \poly_type,crc32 + mvn wCRC,wCRC +.endif + cmp LEN,MIX_BLK_SIZE-1 + adr const_addr, .Lconstants + bls start_final + ld1 {k1k2_v.16b,k3k4_v.16b,poly_v.16b},[const_addr],#48 + movi vzr.16b, #0 + ld1 {k5k0_v.8b,mask_v.8b,fold_poly_v.8b},[const_addr] + +loop_2048: + ld1 {tmp0_v.16b-tmp3_v.16b}, [BUF] + add pmull_blk_ptr,BUF,0x40 + add crc_blk_ptr, BUF,512 + mov tmp4_v.16b,vzr.16b + fmov tmp4_s, wCRC + ldp crc_data0,crc_data1,[crc_blk_ptr],16 + eor tmp0_v.16b,tmp0_v.16b,tmp4_v.16b + mov wCRC, 0 + sub LEN,LEN,MIX_BLK_SIZE + cmp LEN,MIX_BLK_SIZE + ldp crc_data2,crc_data3,[crc_blk_ptr],16 + crc32_u64 wCRC,wCRC,crc_data0 + crc32_u64 wCRC,wCRC,crc_data1 + ldp crc_data0,crc_data1,[crc_blk_ptr],16 + crc32_u64 wCRC,wCRC,crc_data2 + crc32_u64 wCRC,wCRC,crc_data3 + ldp crc_data2,crc_data3,[crc_blk_ptr],16 + + pmull_fold + pmull_fold + pmull_fold + pmull_fold + pmull_fold + pmull_fold + pmull_fold + + /* Folding cache line into 128bit */ + pmull2 tmp4_v.1q, tmp0_v.2d, k3k4_v.2d + crc32_u64 wCRC,wCRC,crc_data0 + crc32_u64 wCRC,wCRC,crc_data1 + ldp crc_data0,crc_data1,[crc_blk_ptr],16 + crc32_u64 wCRC,wCRC,crc_data2 + crc32_u64 wCRC,wCRC,crc_data3 + ldp crc_data2,crc_data3,[crc_blk_ptr],16 + crc32_u64 wCRC,wCRC,crc_data0 + crc32_u64 wCRC,wCRC,crc_data1 + ldp crc_data0,crc_data1,[crc_blk_ptr],16 + pmull tmp0_v.1q, tmp0_v.1d, k3k4_v.1d + crc32_u64 wCRC,wCRC,crc_data2 + crc32_u64 wCRC,wCRC,crc_data3 + ldp crc_data2,crc_data3,[crc_blk_ptr],16 + crc32_u64 wCRC,wCRC,crc_data0 + crc32_u64 wCRC,wCRC,crc_data1 + ldp crc_data0,crc_data1,[crc_blk_ptr],16 + crc32_u64 wCRC,wCRC,crc_data2 + crc32_u64 wCRC,wCRC,crc_data3 + ldp crc_data2,crc_data3,[crc_blk_ptr],16 + eor tmp0_v.16b, tmp0_v.16b, tmp4_v.16b + crc32_u64 wCRC,wCRC,crc_data0 + crc32_u64 wCRC,wCRC,crc_data1 + ldp crc_data0,crc_data1,[crc_blk_ptr],16 + eor tmp0_v.16b, tmp0_v.16b, tmp1_v.16b + crc32_u64 wCRC,wCRC,crc_data2 + crc32_u64 wCRC,wCRC,crc_data3 + ldp crc_data2,crc_data3,[crc_blk_ptr],16 + crc32_u64 wCRC,wCRC,crc_data0 + crc32_u64 wCRC,wCRC,crc_data1 + ldp crc_data0,crc_data1,[crc_blk_ptr],16 + pmull2 tmp4_v.1q, tmp0_v.2d, k3k4_v.2d + crc32_u64 wCRC,wCRC,crc_data2 + crc32_u64 wCRC,wCRC,crc_data3 + ldp crc_data2,crc_data3,[crc_blk_ptr],16 + pmull tmp0_v.1q, tmp0_v.1d, k3k4_v.1d + crc32_u64 wCRC,wCRC,crc_data0 + crc32_u64 wCRC,wCRC,crc_data1 + ldp crc_data0,crc_data1,[crc_blk_ptr],16 + eor tmp0_v.16b, tmp0_v.16b, tmp4_v.16b + crc32_u64 wCRC,wCRC,crc_data2 + crc32_u64 wCRC,wCRC,crc_data3 + ldp crc_data2,crc_data3,[crc_blk_ptr],16 + crc32_u64 wCRC,wCRC,crc_data0 + crc32_u64 wCRC,wCRC,crc_data1 + ldp crc_data0,crc_data1,[crc_blk_ptr],16 + crc32_u64 wCRC,wCRC,crc_data2 + crc32_u64 wCRC,wCRC,crc_data3 + ldp crc_data2,crc_data3,[crc_blk_ptr],16 + eor tmp0_v.16b, tmp0_v.16b, tmp2_v.16b + crc32_u64 wCRC,wCRC,crc_data0 + crc32_u64 wCRC,wCRC,crc_data1 + ldp crc_data0,crc_data1,[crc_blk_ptr],16 + pmull2 tmp4_v.1q, tmp0_v.2d, k3k4_v.2d + crc32_u64 wCRC,wCRC,crc_data2 + crc32_u64 wCRC,wCRC,crc_data3 + ldp crc_data2,crc_data3,[crc_blk_ptr],16 + crc32_u64 wCRC,wCRC,crc_data0 + crc32_u64 wCRC,wCRC,crc_data1 + ldp crc_data0,crc_data1,[crc_blk_ptr],16 + crc32_u64 wCRC,wCRC,crc_data2 + crc32_u64 wCRC,wCRC,crc_data3 + ldp crc_data2,crc_data3,[crc_blk_ptr],16 + pmull tmp0_v.1q, tmp0_v.1d, k3k4_v.1d + crc32_u64 wCRC,wCRC,crc_data0 + crc32_u64 wCRC,wCRC,crc_data1 + ldp crc_data0,crc_data1,[crc_blk_ptr],16 + crc32_u64 wCRC,wCRC,crc_data2 + crc32_u64 wCRC,wCRC,crc_data3 + ldp crc_data2,crc_data3,[crc_blk_ptr],16 + eor tmp0_v.16b, tmp0_v.16b, tmp4_v.16b + crc32_u64 wCRC,wCRC,crc_data0 + crc32_u64 wCRC,wCRC,crc_data1 + ldp crc_data0,crc_data1,[crc_blk_ptr],16 + crc32_u64 wCRC,wCRC,crc_data2 + crc32_u64 wCRC,wCRC,crc_data3 + ldp crc_data2,crc_data3,[crc_blk_ptr],16 + crc32_u64 wCRC,wCRC,crc_data0 + crc32_u64 wCRC,wCRC,crc_data1 + ldp crc_data0,crc_data1,[crc_blk_ptr],16 + eor tmp0_v.16b, tmp0_v.16b, tmp3_v.16b + crc32_u64 wCRC,wCRC,crc_data2 + crc32_u64 wCRC,wCRC,crc_data3 + ldp crc_data2,crc_data3,[crc_blk_ptr],16 + crc32_u64 wCRC,wCRC,crc_data0 + crc32_u64 wCRC,wCRC,crc_data1 + ldp crc_data0,crc_data1,[crc_blk_ptr],16 + + + /** + * perform the last 64 bit fold, also + * adds 32 zeroes to the input stream + */ + ext tmp1_v.16b, tmp0_v.16b, tmp0_v.16b, #8 + crc32_u64 wCRC,wCRC,crc_data2 + crc32_u64 wCRC,wCRC,crc_data3 + ldp crc_data2,crc_data3,[crc_blk_ptr],16 + crc32_u64 wCRC,wCRC,crc_data0 + crc32_u64 wCRC,wCRC,crc_data1 + ldp crc_data0,crc_data1,[crc_blk_ptr],16 + pmull2 tmp1_v.1q, tmp1_v.2d, k3k4_v.2d + crc32_u64 wCRC,wCRC,crc_data2 + crc32_u64 wCRC,wCRC,crc_data3 + ldp crc_data2,crc_data3,[crc_blk_ptr],16 + crc32_u64 wCRC,wCRC,crc_data0 + crc32_u64 wCRC,wCRC,crc_data1 + ldp crc_data0,crc_data1,[crc_blk_ptr],16 + crc32_u64 wCRC,wCRC,crc_data2 + crc32_u64 wCRC,wCRC,crc_data3 + ldp crc_data2,crc_data3,[crc_blk_ptr],16 + ext tmp0_v.16b, tmp0_v.16b, vzr.16b, #8 + crc32_u64 wCRC,wCRC,crc_data0 + crc32_u64 wCRC,wCRC,crc_data1 + ldp crc_data0,crc_data1,[crc_blk_ptr],16 + crc32_u64 wCRC,wCRC,crc_data2 + crc32_u64 wCRC,wCRC,crc_data3 + ldp crc_data2,crc_data3,[crc_blk_ptr],16 + eor tmp0_v.16b, tmp0_v.16b, tmp1_v.16b + crc32_u64 wCRC,wCRC,crc_data0 + crc32_u64 wCRC,wCRC,crc_data1 + ldp crc_data0,crc_data1,[crc_blk_ptr],16 + crc32_u64 wCRC,wCRC,crc_data2 + crc32_u64 wCRC,wCRC,crc_data3 + ldp crc_data2,crc_data3,[crc_blk_ptr],16 + + /* final 32-bit fold */ + ext tmp1_v.16b, tmp0_v.16b, vzr.16b, #4 + and tmp0_v.16b, tmp0_v.16b, mask_v.16b + crc32_u64 wCRC,wCRC,crc_data0 + crc32_u64 wCRC,wCRC,crc_data1 + ldp crc_data0,crc_data1,[crc_blk_ptr],16 + pmull tmp0_v.1q, tmp0_v.1d, k5k0_v.1d + crc32_u64 wCRC,wCRC,crc_data2 + crc32_u64 wCRC,wCRC,crc_data3 + ldp crc_data2,crc_data3,[crc_blk_ptr],16 + crc32_u64 wCRC,wCRC,crc_data0 + crc32_u64 wCRC,wCRC,crc_data1 + ldp crc_data0,crc_data1,[crc_blk_ptr],16 + crc32_u64 wCRC,wCRC,crc_data2 + crc32_u64 wCRC,wCRC,crc_data3 + ldp crc_data2,crc_data3,[crc_blk_ptr],16 + eor tmp0_v.16b, tmp0_v.16b, tmp1_v.16b + + /** + * Finish up with the bit-reversed barrett + * reduction 64 ==> 32 bits + */ + crc32_u64 wCRC,wCRC,crc_data0 + crc32_u64 wCRC,wCRC,crc_data1 + and tmp1_v.16b, tmp0_v.16b, mask_v.16b + ldp crc_data0,crc_data1,[crc_blk_ptr],16 + ext tmp1_v.16b, vzr.16b, tmp1_v.16b, #8 + crc32_u64 wCRC,wCRC,crc_data2 + crc32_u64 wCRC,wCRC,crc_data3 + pmull2 tmp1_v.1q, tmp1_v.2d, poly_v.2d + ldp crc_data2,crc_data3,[crc_blk_ptr],16 + crc32_u64 wCRC,wCRC,crc_data0 + crc32_u64 wCRC,wCRC,crc_data1 + ldp crc_data0,crc_data1,[crc_blk_ptr],16 + crc32_u64 wCRC,wCRC,crc_data2 + crc32_u64 wCRC,wCRC,crc_data3 + and tmp1_v.16b, tmp1_v.16b, mask_v.16b + ldp crc_data2,crc_data3,[crc_blk_ptr],16 + pmull tmp1_v.1q, tmp1_v.1d, poly_v.1d + crc32_u64 wCRC,wCRC,crc_data0 + crc32_u64 wCRC,wCRC,crc_data1 + ldp crc_data0,crc_data1,[crc_blk_ptr],16 + eor tmp0_v.16b, tmp0_v.16b, tmp1_v.16b + crc32_u64 wCRC,wCRC,crc_data2 + crc32_u64 wCRC,wCRC,crc_data3 + mov tmp4_v.16b,vzr.16b + mov tmp4_v.s[0], tmp0_v.s[1] + ldp crc_data2,crc_data3,[crc_blk_ptr],16 + crc32_u64 wCRC,wCRC,crc_data0 + crc32_u64 wCRC,wCRC,crc_data1 + ldp crc_data0,crc_data1,[crc_blk_ptr],16 + crc32_u64 wCRC,wCRC,crc_data2 + crc32_u64 wCRC,wCRC,crc_data3 + ldp crc_data2,crc_data3,[crc_blk_ptr],16 + crc32_u64 wCRC,wCRC,crc_data0 + crc32_u64 wCRC,wCRC,crc_data1 + ldp crc_data0,crc_data1,[crc_blk_ptr],16 + crc32_u64 wCRC,wCRC,crc_data2 + crc32_u64 wCRC,wCRC,crc_data3 + ldp crc_data2,crc_data3,[crc_blk_ptr],16 + crc32_u64 wCRC,wCRC,crc_data0 + crc32_u64 wCRC,wCRC,crc_data1 + ldp crc_data0,crc_data1,[crc_blk_ptr],16 + crc32_u64 wCRC,wCRC,crc_data2 + crc32_u64 wCRC,wCRC,crc_data3 + ldp crc_data2,crc_data3,[crc_blk_ptr],16 + + crc32_u64 wCRC,wCRC,crc_data0 + crc32_u64 wCRC,wCRC,crc_data1 + crc32_u64 wCRC,wCRC,crc_data2 + crc32_u64 wCRC,wCRC,crc_data3 + + pmull tmp4_v.1q, tmp4_v.1d, fold_poly_v.1d + add BUF,BUF,MIX_BLK_SIZE + fmov xPmull, tmp4_d + crc32_u64 wPmull, wzr, xPmull + eor wCRC, wPmull, wCRC + bge loop_2048 +start_final: + cmp LEN, 63 + bls .loop_16B +.loop_64B: + ldp data0, data1, [BUF],#16 + sub LEN,LEN,#64 + ldp data2, data3, [BUF],#16 + cmp LEN,#64 + crc32_u64 wCRC, wCRC, data0 + crc32_u64 wCRC, wCRC, data1 + ldp data0, data1, [BUF],#16 + crc32_u64 wCRC, wCRC, data2 + crc32_u64 wCRC, wCRC, data3 + ldp data2, data3, [BUF],#16 + crc32_u64 wCRC, wCRC, data0 + crc32_u64 wCRC, wCRC, data1 + crc32_u64 wCRC, wCRC, data2 + crc32_u64 wCRC, wCRC, data3 + bge .loop_64B + +.loop_16B: + cmp LEN, 15 + bls .less_16B + ldp data0, data1, [BUF],#16 + sub LEN,LEN,#16 + cmp LEN,15 + crc32_u64 wCRC, wCRC, data0 + crc32_u64 wCRC, wCRC, data1 + bls .less_16B + ldp data0, data1, [BUF],#16 + sub LEN,LEN,#16 + cmp LEN,15 + crc32_u64 wCRC, wCRC, data0 + crc32_u64 wCRC, wCRC, data1 + bls .less_16B + ldp data0, data1, [BUF],#16 + sub LEN,LEN,#16 //MUST less than 16B + crc32_u64 wCRC, wCRC, data0 + crc32_u64 wCRC, wCRC, data1 +.less_16B: + cmp LEN, 7 + bls .less_8B + ldr data0, [BUF], 8 + sub LEN, LEN, #8 + crc32_u64 wCRC, wCRC, data0 +.less_8B: + cmp LEN, 3 + bls .less_4B + ldr wdata, [BUF], 4 + sub LEN, LEN, #4 + crc32_u32 wCRC, wCRC, wdata +.less_4B: + cmp LEN, 1 + bls .less_2B + ldrh wdata, [BUF], 2 + sub LEN, LEN, #2 + crc32_u16 wCRC, wCRC, wdata +.less_2B: + cbz LEN, .finish_exit + ldrb wdata, [BUF] + crc32_u8 wCRC, wCRC, wdata +.finish_exit: +.ifc \poly_type,crc32 + mvn w0, wCRC +.else + mov w0, wCRC +.endif + ret +.endm + diff --git a/src/isa-l/crc/aarch64/crc32_gzip_refl_3crc_fold.S b/src/isa-l/crc/aarch64/crc32_gzip_refl_3crc_fold.S new file mode 100644 index 000000000..116d62cc9 --- /dev/null +++ b/src/isa-l/crc/aarch64/crc32_gzip_refl_3crc_fold.S @@ -0,0 +1,95 @@ +######################################################################## +# Copyright(c) 2020 Arm Corporation All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Arm Corporation nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +######################################################################### +#include "crc32_aarch64_common.h" + .text + .align 6 + .arch armv8-a+crc+crypto +.macro crc32_u64 dst,src,data + crc32x \dst,\src,\data +.endm +.macro crc32_u32 dst,src,data + crc32w \dst,\src,\data +.endm +.macro crc32_u16 dst,src,data + crc32h \dst,\src,\data +.endm +.macro crc32_u8 dst,src,data + crc32b \dst,\src,\data +.endm +.macro declare_var_vector_reg name:req,reg:req + q\name .req q\reg + v\name .req v\reg + s\name .req s\reg + d\name .req d\reg +.endm + + BUF .req x1 + ptr_crc0 .req x1 + LEN .req x2 + wCRC .req w0 + crc0 .req w0 + xcrc0 .req x0 + + crc1 .req w3 + crc2 .req w4 + xcrc1 .req x3 + const_adr .req x3 + ptr_crc1 .req x6 + ptr_crc2 .req x7 + crc0_data0 .req x9 + crc0_data1 .req x10 + crc1_data0 .req x11 + crc1_data1 .req x12 + crc2_data0 .req x13 + crc2_data1 .req x14 + + wdata .req w3 + data0 .req x3 + data1 .req x4 + data2 .req x5 + data3 .req x6 + + declare_var_vector_reg tmp0,0 + declare_var_vector_reg tmp1,1 + declare_var_vector_reg const0,2 + declare_var_vector_reg const1,3 + +/** + uint32_t crc32_gzip_refl( + uint32_t wCRC, + const unsigned char *BUF, + uint64_t LEN + ); +*/ + + .global crc32_gzip_refl_3crc_fold + .type crc32_gzip_refl_3crc_fold, %function +crc32_gzip_refl_3crc_fold: + crc32_3crc_fold crc32 + .size crc32_gzip_refl_3crc_fold, .-crc32_gzip_refl_3crc_fold diff --git a/src/isa-l/crc/aarch64/crc32_gzip_refl_crc_ext.S b/src/isa-l/crc/aarch64/crc32_gzip_refl_crc_ext.S new file mode 100644 index 000000000..8e3d227be --- /dev/null +++ b/src/isa-l/crc/aarch64/crc32_gzip_refl_crc_ext.S @@ -0,0 +1,66 @@ +/********************************************************************** + Copyright(c) 2020 Arm Corporation All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Arm Corporation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +**********************************************************************/ + + .text + .align 6 + .arch armv8-a+crc + + +#include "crc32_aarch64_common.h" + + BUF .req x1 + LEN .req x2 + wCRC .req w0 + data0 .req x4 + data1 .req x5 + data2 .req x6 + data3 .req x7 + wdata .req w3 +.macro crc32_u64 dst,src,data + crc32x \dst,\src,\data +.endm +.macro crc32_u32 dst,src,data + crc32w \dst,\src,\data +.endm +.macro crc32_u16 dst,src,data + crc32h \dst,\src,\data +.endm +.macro crc32_u8 dst,src,data + crc32b \dst,\src,\data +.endm + + /** + * uint32_t crc32_gzip_refl_crc_ext(const unsigned char *BUF, + * uint64_t LEN,uint32_t wCRC); + */ + .global crc32_gzip_refl_crc_ext + .type crc32_gzip_refl_crc_ext, %function +crc32_gzip_refl_crc_ext: + crc32_hw_common crc32 + .size crc32_gzip_refl_crc_ext, .-crc32_gzip_refl_crc_ext diff --git a/src/isa-l/crc/aarch64/crc32_gzip_refl_pmull.S b/src/isa-l/crc/aarch64/crc32_gzip_refl_pmull.S new file mode 100644 index 000000000..d52e2d8f5 --- /dev/null +++ b/src/isa-l/crc/aarch64/crc32_gzip_refl_pmull.S @@ -0,0 +1,33 @@ +######################################################################## +# Copyright(c) 2019 Arm Corporation All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Arm Corporation nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +######################################################################### + +#include "crc32_gzip_refl_pmull.h" +#include "crc32_refl_common_pmull.h" + +crc32_refl_func crc32_gzip_refl_pmull diff --git a/src/isa-l/crc/aarch64/crc32_gzip_refl_pmull.h b/src/isa-l/crc/aarch64/crc32_gzip_refl_pmull.h new file mode 100644 index 000000000..883567d97 --- /dev/null +++ b/src/isa-l/crc/aarch64/crc32_gzip_refl_pmull.h @@ -0,0 +1,87 @@ +######################################################################## +# Copyright(c) 2019 Arm Corporation All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Arm Corporation nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +######################################################################### + +.equ p4_low_b0, 0x2d95 +.equ p4_low_b1, 0x8f35 +.equ p4_high_b0, 0x13d7 +.equ p4_high_b1, 0x1d95 +.equ p1_low_b0, 0x9191 +.equ p1_low_b1, 0xae68 +.equ p1_high_b0, 0x009e +.equ p1_high_b1, 0xccaa +.equ p0_low_b0, 0x6765 +.equ p0_low_b1, 0xb8bc +.equ p0_high_b0, p1_high_b0 +.equ p0_high_b1, p1_high_b1 +.equ br_low_b0, 0x0641 +.equ br_low_b1, 0xdb71 +.equ br_low_b2, 0x1 +.equ br_high_b0, 0x1641 +.equ br_high_b1, 0xf701 +.equ br_high_b2, 0x1 + + .text + .section .rodata + .align 4 + .set .lanchor_crc_tab,. + 0 + .type crc32_table_gzip_refl, %object + .size crc32_table_gzip_refl, 1024 +crc32_table_gzip_refl: + .word 0x00000000, 0x77073096, 0xee0e612c, 0x990951ba, 0x076dc419, 0x706af48f, 0xe963a535, 0x9e6495a3 + .word 0x0edb8832, 0x79dcb8a4, 0xe0d5e91e, 0x97d2d988, 0x09b64c2b, 0x7eb17cbd, 0xe7b82d07, 0x90bf1d91 + .word 0x1db71064, 0x6ab020f2, 0xf3b97148, 0x84be41de, 0x1adad47d, 0x6ddde4eb, 0xf4d4b551, 0x83d385c7 + .word 0x136c9856, 0x646ba8c0, 0xfd62f97a, 0x8a65c9ec, 0x14015c4f, 0x63066cd9, 0xfa0f3d63, 0x8d080df5 + .word 0x3b6e20c8, 0x4c69105e, 0xd56041e4, 0xa2677172, 0x3c03e4d1, 0x4b04d447, 0xd20d85fd, 0xa50ab56b + .word 0x35b5a8fa, 0x42b2986c, 0xdbbbc9d6, 0xacbcf940, 0x32d86ce3, 0x45df5c75, 0xdcd60dcf, 0xabd13d59 + .word 0x26d930ac, 0x51de003a, 0xc8d75180, 0xbfd06116, 0x21b4f4b5, 0x56b3c423, 0xcfba9599, 0xb8bda50f + .word 0x2802b89e, 0x5f058808, 0xc60cd9b2, 0xb10be924, 0x2f6f7c87, 0x58684c11, 0xc1611dab, 0xb6662d3d + .word 0x76dc4190, 0x01db7106, 0x98d220bc, 0xefd5102a, 0x71b18589, 0x06b6b51f, 0x9fbfe4a5, 0xe8b8d433 + .word 0x7807c9a2, 0x0f00f934, 0x9609a88e, 0xe10e9818, 0x7f6a0dbb, 0x086d3d2d, 0x91646c97, 0xe6635c01 + .word 0x6b6b51f4, 0x1c6c6162, 0x856530d8, 0xf262004e, 0x6c0695ed, 0x1b01a57b, 0x8208f4c1, 0xf50fc457 + .word 0x65b0d9c6, 0x12b7e950, 0x8bbeb8ea, 0xfcb9887c, 0x62dd1ddf, 0x15da2d49, 0x8cd37cf3, 0xfbd44c65 + .word 0x4db26158, 0x3ab551ce, 0xa3bc0074, 0xd4bb30e2, 0x4adfa541, 0x3dd895d7, 0xa4d1c46d, 0xd3d6f4fb + .word 0x4369e96a, 0x346ed9fc, 0xad678846, 0xda60b8d0, 0x44042d73, 0x33031de5, 0xaa0a4c5f, 0xdd0d7cc9 + .word 0x5005713c, 0x270241aa, 0xbe0b1010, 0xc90c2086, 0x5768b525, 0x206f85b3, 0xb966d409, 0xce61e49f + .word 0x5edef90e, 0x29d9c998, 0xb0d09822, 0xc7d7a8b4, 0x59b33d17, 0x2eb40d81, 0xb7bd5c3b, 0xc0ba6cad + .word 0xedb88320, 0x9abfb3b6, 0x03b6e20c, 0x74b1d29a, 0xead54739, 0x9dd277af, 0x04db2615, 0x73dc1683 + .word 0xe3630b12, 0x94643b84, 0x0d6d6a3e, 0x7a6a5aa8, 0xe40ecf0b, 0x9309ff9d, 0x0a00ae27, 0x7d079eb1 + .word 0xf00f9344, 0x8708a3d2, 0x1e01f268, 0x6906c2fe, 0xf762575d, 0x806567cb, 0x196c3671, 0x6e6b06e7 + .word 0xfed41b76, 0x89d32be0, 0x10da7a5a, 0x67dd4acc, 0xf9b9df6f, 0x8ebeeff9, 0x17b7be43, 0x60b08ed5 + .word 0xd6d6a3e8, 0xa1d1937e, 0x38d8c2c4, 0x4fdff252, 0xd1bb67f1, 0xa6bc5767, 0x3fb506dd, 0x48b2364b + .word 0xd80d2bda, 0xaf0a1b4c, 0x36034af6, 0x41047a60, 0xdf60efc3, 0xa867df55, 0x316e8eef, 0x4669be79 + .word 0xcb61b38c, 0xbc66831a, 0x256fd2a0, 0x5268e236, 0xcc0c7795, 0xbb0b4703, 0x220216b9, 0x5505262f + .word 0xc5ba3bbe, 0xb2bd0b28, 0x2bb45a92, 0x5cb36a04, 0xc2d7ffa7, 0xb5d0cf31, 0x2cd99e8b, 0x5bdeae1d + .word 0x9b64c2b0, 0xec63f226, 0x756aa39c, 0x026d930a, 0x9c0906a9, 0xeb0e363f, 0x72076785, 0x05005713 + .word 0x95bf4a82, 0xe2b87a14, 0x7bb12bae, 0x0cb61b38, 0x92d28e9b, 0xe5d5be0d, 0x7cdcefb7, 0x0bdbdf21 + .word 0x86d3d2d4, 0xf1d4e242, 0x68ddb3f8, 0x1fda836e, 0x81be16cd, 0xf6b9265b, 0x6fb077e1, 0x18b74777 + .word 0x88085ae6, 0xff0f6a70, 0x66063bca, 0x11010b5c, 0x8f659eff, 0xf862ae69, 0x616bffd3, 0x166ccf45 + .word 0xa00ae278, 0xd70dd2ee, 0x4e048354, 0x3903b3c2, 0xa7672661, 0xd06016f7, 0x4969474d, 0x3e6e77db + .word 0xaed16a4a, 0xd9d65adc, 0x40df0b66, 0x37d83bf0, 0xa9bcae53, 0xdebb9ec5, 0x47b2cf7f, 0x30b5ffe9 + .word 0xbdbdf21c, 0xcabac28a, 0x53b39330, 0x24b4a3a6, 0xbad03605, 0xcdd70693, 0x54de5729, 0x23d967bf + .word 0xb3667a2e, 0xc4614ab8, 0x5d681b02, 0x2a6f2b94, 0xb40bbe37, 0xc30c8ea1, 0x5a05df1b, 0x2d02ef8d diff --git a/src/isa-l/crc/aarch64/crc32_ieee_norm_pmull.S b/src/isa-l/crc/aarch64/crc32_ieee_norm_pmull.S new file mode 100644 index 000000000..32966fb9d --- /dev/null +++ b/src/isa-l/crc/aarch64/crc32_ieee_norm_pmull.S @@ -0,0 +1,33 @@ +######################################################################## +# Copyright(c) 2019 Arm Corporation All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Arm Corporation nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +######################################################################### + +#include "crc32_ieee_norm_pmull.h" +#include "crc32_norm_common_pmull.h" + +crc32_norm_func crc32_ieee_norm_pmull diff --git a/src/isa-l/crc/aarch64/crc32_ieee_norm_pmull.h b/src/isa-l/crc/aarch64/crc32_ieee_norm_pmull.h new file mode 100644 index 000000000..67acd2a03 --- /dev/null +++ b/src/isa-l/crc/aarch64/crc32_ieee_norm_pmull.h @@ -0,0 +1,87 @@ +######################################################################## +# Copyright(c) 2019 Arm Corporation All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Arm Corporation nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +######################################################################### + +.equ p4_low_b0, 0x8b11 +.equ p4_low_b1, 0xe622 +.equ p4_high_b0, 0x794c +.equ p4_high_b1, 0x8833 +.equ p1_low_b0, 0x5605 +.equ p1_low_b1, 0xe8a4 +.equ p1_high_b0, 0xcd4c +.equ p1_high_b1, 0xc5b9 +.equ p0_low_b0, 0x678d +.equ p0_low_b1, 0x490d +.equ p0_high_b0, 0xaa66 +.equ p0_high_b1, 0xf200 +.equ br_low_b0, 0x01df +.equ br_low_b1, 0x04d1 +.equ br_low_b2, 0x1 +.equ br_high_b0, 0x1db7 +.equ br_high_b1, 0x04c1 +.equ br_high_b2, 0x1 + + .text + .section .rodata + .align 4 + .set .lanchor_crc_tab,. + 0 + .type crc32_table_ieee_norm, %object + .size crc32_table_ieee_norm, 1024 +crc32_table_ieee_norm: + .word 0x00000000, 0x04c11db7, 0x09823b6e, 0x0d4326d9, 0x130476dc, 0x17c56b6b, 0x1a864db2, 0x1e475005 + .word 0x2608edb8, 0x22c9f00f, 0x2f8ad6d6, 0x2b4bcb61, 0x350c9b64, 0x31cd86d3, 0x3c8ea00a, 0x384fbdbd + .word 0x4c11db70, 0x48d0c6c7, 0x4593e01e, 0x4152fda9, 0x5f15adac, 0x5bd4b01b, 0x569796c2, 0x52568b75 + .word 0x6a1936c8, 0x6ed82b7f, 0x639b0da6, 0x675a1011, 0x791d4014, 0x7ddc5da3, 0x709f7b7a, 0x745e66cd + .word 0x9823b6e0, 0x9ce2ab57, 0x91a18d8e, 0x95609039, 0x8b27c03c, 0x8fe6dd8b, 0x82a5fb52, 0x8664e6e5 + .word 0xbe2b5b58, 0xbaea46ef, 0xb7a96036, 0xb3687d81, 0xad2f2d84, 0xa9ee3033, 0xa4ad16ea, 0xa06c0b5d + .word 0xd4326d90, 0xd0f37027, 0xddb056fe, 0xd9714b49, 0xc7361b4c, 0xc3f706fb, 0xceb42022, 0xca753d95 + .word 0xf23a8028, 0xf6fb9d9f, 0xfbb8bb46, 0xff79a6f1, 0xe13ef6f4, 0xe5ffeb43, 0xe8bccd9a, 0xec7dd02d + .word 0x34867077, 0x30476dc0, 0x3d044b19, 0x39c556ae, 0x278206ab, 0x23431b1c, 0x2e003dc5, 0x2ac12072 + .word 0x128e9dcf, 0x164f8078, 0x1b0ca6a1, 0x1fcdbb16, 0x018aeb13, 0x054bf6a4, 0x0808d07d, 0x0cc9cdca + .word 0x7897ab07, 0x7c56b6b0, 0x71159069, 0x75d48dde, 0x6b93dddb, 0x6f52c06c, 0x6211e6b5, 0x66d0fb02 + .word 0x5e9f46bf, 0x5a5e5b08, 0x571d7dd1, 0x53dc6066, 0x4d9b3063, 0x495a2dd4, 0x44190b0d, 0x40d816ba + .word 0xaca5c697, 0xa864db20, 0xa527fdf9, 0xa1e6e04e, 0xbfa1b04b, 0xbb60adfc, 0xb6238b25, 0xb2e29692 + .word 0x8aad2b2f, 0x8e6c3698, 0x832f1041, 0x87ee0df6, 0x99a95df3, 0x9d684044, 0x902b669d, 0x94ea7b2a + .word 0xe0b41de7, 0xe4750050, 0xe9362689, 0xedf73b3e, 0xf3b06b3b, 0xf771768c, 0xfa325055, 0xfef34de2 + .word 0xc6bcf05f, 0xc27dede8, 0xcf3ecb31, 0xcbffd686, 0xd5b88683, 0xd1799b34, 0xdc3abded, 0xd8fba05a + .word 0x690ce0ee, 0x6dcdfd59, 0x608edb80, 0x644fc637, 0x7a089632, 0x7ec98b85, 0x738aad5c, 0x774bb0eb + .word 0x4f040d56, 0x4bc510e1, 0x46863638, 0x42472b8f, 0x5c007b8a, 0x58c1663d, 0x558240e4, 0x51435d53 + .word 0x251d3b9e, 0x21dc2629, 0x2c9f00f0, 0x285e1d47, 0x36194d42, 0x32d850f5, 0x3f9b762c, 0x3b5a6b9b + .word 0x0315d626, 0x07d4cb91, 0x0a97ed48, 0x0e56f0ff, 0x1011a0fa, 0x14d0bd4d, 0x19939b94, 0x1d528623 + .word 0xf12f560e, 0xf5ee4bb9, 0xf8ad6d60, 0xfc6c70d7, 0xe22b20d2, 0xe6ea3d65, 0xeba91bbc, 0xef68060b + .word 0xd727bbb6, 0xd3e6a601, 0xdea580d8, 0xda649d6f, 0xc423cd6a, 0xc0e2d0dd, 0xcda1f604, 0xc960ebb3 + .word 0xbd3e8d7e, 0xb9ff90c9, 0xb4bcb610, 0xb07daba7, 0xae3afba2, 0xaafbe615, 0xa7b8c0cc, 0xa379dd7b + .word 0x9b3660c6, 0x9ff77d71, 0x92b45ba8, 0x9675461f, 0x8832161a, 0x8cf30bad, 0x81b02d74, 0x857130c3 + .word 0x5d8a9099, 0x594b8d2e, 0x5408abf7, 0x50c9b640, 0x4e8ee645, 0x4a4ffbf2, 0x470cdd2b, 0x43cdc09c + .word 0x7b827d21, 0x7f436096, 0x7200464f, 0x76c15bf8, 0x68860bfd, 0x6c47164a, 0x61043093, 0x65c52d24 + .word 0x119b4be9, 0x155a565e, 0x18197087, 0x1cd86d30, 0x029f3d35, 0x065e2082, 0x0b1d065b, 0x0fdc1bec + .word 0x3793a651, 0x3352bbe6, 0x3e119d3f, 0x3ad08088, 0x2497d08d, 0x2056cd3a, 0x2d15ebe3, 0x29d4f654 + .word 0xc5a92679, 0xc1683bce, 0xcc2b1d17, 0xc8ea00a0, 0xd6ad50a5, 0xd26c4d12, 0xdf2f6bcb, 0xdbee767c + .word 0xe3a1cbc1, 0xe760d676, 0xea23f0af, 0xeee2ed18, 0xf0a5bd1d, 0xf464a0aa, 0xf9278673, 0xfde69bc4 + .word 0x89b8fd09, 0x8d79e0be, 0x803ac667, 0x84fbdbd0, 0x9abc8bd5, 0x9e7d9662, 0x933eb0bb, 0x97ffad0c + .word 0xafb010b1, 0xab710d06, 0xa6322bdf, 0xa2f33668, 0xbcb4666d, 0xb8757bda, 0xb5365d03, 0xb1f740b4 diff --git a/src/isa-l/crc/aarch64/crc32_iscsi_3crc_fold.S b/src/isa-l/crc/aarch64/crc32_iscsi_3crc_fold.S new file mode 100644 index 000000000..2beaa80c7 --- /dev/null +++ b/src/isa-l/crc/aarch64/crc32_iscsi_3crc_fold.S @@ -0,0 +1,97 @@ +######################################################################## +# Copyright(c) 2020 Arm Corporation All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Arm Corporation nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +######################################################################### + + + .text + .align 6 + .arch armv8-a+crc+crypto +#include "crc32_aarch64_common.h" +.macro crc32_u64 dst,src,data + crc32cx \dst,\src,\data +.endm +.macro crc32_u32 dst,src,data + crc32cw \dst,\src,\data +.endm +.macro crc32_u16 dst,src,data + crc32ch \dst,\src,\data +.endm +.macro crc32_u8 dst,src,data + crc32cb \dst,\src,\data +.endm +.macro declare_var_vector_reg name:req,reg:req + q\name .req q\reg + v\name .req v\reg + s\name .req s\reg + d\name .req d\reg +.endm + + BUF .req x0 + LEN .req x1 + wCRC .req w2 + crc0 .req w2 + crc1 .req w3 + crc2 .req w4 + xcrc0 .req x2 + xcrc1 .req x3 + const_adr .req x3 + ptr_crc0 .req x0 + ptr_crc1 .req x6 + ptr_crc2 .req x7 + crc0_data0 .req x9 + crc0_data1 .req x10 + crc1_data0 .req x11 + crc1_data1 .req x12 + crc2_data0 .req x13 + crc2_data1 .req x14 + + wdata .req w3 + data0 .req x3 + data1 .req x4 + data2 .req x5 + data3 .req x6 + + declare_var_vector_reg tmp0,0 + declare_var_vector_reg tmp1,1 + declare_var_vector_reg const0,2 + declare_var_vector_reg const1,3 + +/** + unsigned int crc32_iscsi( + unsigned char *BUF, + int LEN, + unsigned int wCRC + ); + +*/ + + .global crc32_iscsi_3crc_fold + .type crc32_iscsi_3crc_fold, %function +crc32_iscsi_3crc_fold: + crc32_3crc_fold crc32c + .size crc32_iscsi_3crc_fold, .-crc32_iscsi_3crc_fold diff --git a/src/isa-l/crc/aarch64/crc32_iscsi_crc_ext.S b/src/isa-l/crc/aarch64/crc32_iscsi_crc_ext.S new file mode 100644 index 000000000..359401a52 --- /dev/null +++ b/src/isa-l/crc/aarch64/crc32_iscsi_crc_ext.S @@ -0,0 +1,65 @@ +/********************************************************************** + Copyright(c) 2020 Arm Corporation All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Arm Corporation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +**********************************************************************/ + + .text + .align 6 + .arch armv8-a+crc + + +#include "crc32_aarch64_common.h" + BUF .req x0 + LEN .req x1 + wCRC .req w2 + data0 .req x4 + data1 .req x5 + data2 .req x6 + data3 .req x7 + wdata .req w3 +.macro crc32_u64 dst,src,data + crc32cx \dst,\src,\data +.endm +.macro crc32_u32 dst,src,data + crc32cw \dst,\src,\data +.endm +.macro crc32_u16 dst,src,data + crc32ch \dst,\src,\data +.endm +.macro crc32_u8 dst,src,data + crc32cb \dst,\src,\data +.endm + + /** + * uint32_t crc32_iscsi_crc_ext(const unsigned char *BUF, + * uint64_t LEN,uint32_t wCRC); + */ + .global crc32_iscsi_crc_ext + .type crc32_iscsi_crc_ext, %function +crc32_iscsi_crc_ext: + crc32_hw_common crc32c + .size crc32_iscsi_crc_ext, .-crc32_iscsi_crc_ext diff --git a/src/isa-l/crc/aarch64/crc32_iscsi_refl_pmull.S b/src/isa-l/crc/aarch64/crc32_iscsi_refl_pmull.S new file mode 100644 index 000000000..09a88e2e1 --- /dev/null +++ b/src/isa-l/crc/aarch64/crc32_iscsi_refl_pmull.S @@ -0,0 +1,53 @@ +######################################################################## +# Copyright(c) 2019 Arm Corporation All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Arm Corporation nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +######################################################################### + +#include "crc32_iscsi_refl_pmull.h" +#include "crc32_refl_common_pmull.h" + +crc32_refl_func crc32_iscsi_refl_pmull_internal + + .arch armv8-a+crc+crypto + .text + .align 3 + .global crc32_iscsi_refl_pmull + .type crc32_iscsi_refl_pmull, %function +crc32_iscsi_refl_pmull: + stp x29, x30, [sp, -32]! + mov x29, sp + + mov w7, w2 + sxtw x2, w1 + mov x1, x0 + mov w0, w7 + mvn w0, w0 + + bl crc32_iscsi_refl_pmull_internal + mvn w0, w0 + ldp x29, x30, [sp], 32 + ret diff --git a/src/isa-l/crc/aarch64/crc32_iscsi_refl_pmull.h b/src/isa-l/crc/aarch64/crc32_iscsi_refl_pmull.h new file mode 100644 index 000000000..c17b91be3 --- /dev/null +++ b/src/isa-l/crc/aarch64/crc32_iscsi_refl_pmull.h @@ -0,0 +1,87 @@ +######################################################################## +# Copyright(c) 2019 Arm Corporation All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Arm Corporation nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +######################################################################### + +.equ p4_low_b0, 0xef02 +.equ p4_low_b1, 0x740e +.equ p4_high_b0, 0xddf8 +.equ p4_high_b1, 0x9e4a +.equ p1_low_b0, 0x0dfe +.equ p1_low_b1, 0xf20c +.equ p1_high_b0, 0x7d27 +.equ p1_high_b1, 0x493c +.equ p0_low_b0, 0xaab8 +.equ p0_low_b1, 0xdd45 +.equ p0_high_b0, p1_high_b0 +.equ p0_high_b1, p1_high_b1 +.equ br_low_b0, 0x76f1 +.equ br_low_b1, 0x05ec +.equ br_low_b2, 0x1 +.equ br_high_b0, 0x13f1 +.equ br_high_b1, 0xdea7 +.equ br_high_b2, 0x0 + + .text + .section .rodata + .align 4 + .set .lanchor_crc_tab,. + 0 + .type crc32_table_iscsi_refl, %object + .size crc32_table_iscsi_refl, 1024 +crc32_table_iscsi_refl: + .word 0x00000000, 0xF26B8303, 0xE13B70F7, 0x1350F3F4, 0xC79A971F, 0x35F1141C, 0x26A1E7E8, 0xD4CA64EB + .word 0x8AD958CF, 0x78B2DBCC, 0x6BE22838, 0x9989AB3B, 0x4D43CFD0, 0xBF284CD3, 0xAC78BF27, 0x5E133C24 + .word 0x105EC76F, 0xE235446C, 0xF165B798, 0x030E349B, 0xD7C45070, 0x25AFD373, 0x36FF2087, 0xC494A384 + .word 0x9A879FA0, 0x68EC1CA3, 0x7BBCEF57, 0x89D76C54, 0x5D1D08BF, 0xAF768BBC, 0xBC267848, 0x4E4DFB4B + .word 0x20BD8EDE, 0xD2D60DDD, 0xC186FE29, 0x33ED7D2A, 0xE72719C1, 0x154C9AC2, 0x061C6936, 0xF477EA35 + .word 0xAA64D611, 0x580F5512, 0x4B5FA6E6, 0xB93425E5, 0x6DFE410E, 0x9F95C20D, 0x8CC531F9, 0x7EAEB2FA + .word 0x30E349B1, 0xC288CAB2, 0xD1D83946, 0x23B3BA45, 0xF779DEAE, 0x05125DAD, 0x1642AE59, 0xE4292D5A + .word 0xBA3A117E, 0x4851927D, 0x5B016189, 0xA96AE28A, 0x7DA08661, 0x8FCB0562, 0x9C9BF696, 0x6EF07595 + .word 0x417B1DBC, 0xB3109EBF, 0xA0406D4B, 0x522BEE48, 0x86E18AA3, 0x748A09A0, 0x67DAFA54, 0x95B17957 + .word 0xCBA24573, 0x39C9C670, 0x2A993584, 0xD8F2B687, 0x0C38D26C, 0xFE53516F, 0xED03A29B, 0x1F682198 + .word 0x5125DAD3, 0xA34E59D0, 0xB01EAA24, 0x42752927, 0x96BF4DCC, 0x64D4CECF, 0x77843D3B, 0x85EFBE38 + .word 0xDBFC821C, 0x2997011F, 0x3AC7F2EB, 0xC8AC71E8, 0x1C661503, 0xEE0D9600, 0xFD5D65F4, 0x0F36E6F7 + .word 0x61C69362, 0x93AD1061, 0x80FDE395, 0x72966096, 0xA65C047D, 0x5437877E, 0x4767748A, 0xB50CF789 + .word 0xEB1FCBAD, 0x197448AE, 0x0A24BB5A, 0xF84F3859, 0x2C855CB2, 0xDEEEDFB1, 0xCDBE2C45, 0x3FD5AF46 + .word 0x7198540D, 0x83F3D70E, 0x90A324FA, 0x62C8A7F9, 0xB602C312, 0x44694011, 0x5739B3E5, 0xA55230E6 + .word 0xFB410CC2, 0x092A8FC1, 0x1A7A7C35, 0xE811FF36, 0x3CDB9BDD, 0xCEB018DE, 0xDDE0EB2A, 0x2F8B6829 + .word 0x82F63B78, 0x709DB87B, 0x63CD4B8F, 0x91A6C88C, 0x456CAC67, 0xB7072F64, 0xA457DC90, 0x563C5F93 + .word 0x082F63B7, 0xFA44E0B4, 0xE9141340, 0x1B7F9043, 0xCFB5F4A8, 0x3DDE77AB, 0x2E8E845F, 0xDCE5075C + .word 0x92A8FC17, 0x60C37F14, 0x73938CE0, 0x81F80FE3, 0x55326B08, 0xA759E80B, 0xB4091BFF, 0x466298FC + .word 0x1871A4D8, 0xEA1A27DB, 0xF94AD42F, 0x0B21572C, 0xDFEB33C7, 0x2D80B0C4, 0x3ED04330, 0xCCBBC033 + .word 0xA24BB5A6, 0x502036A5, 0x4370C551, 0xB11B4652, 0x65D122B9, 0x97BAA1BA, 0x84EA524E, 0x7681D14D + .word 0x2892ED69, 0xDAF96E6A, 0xC9A99D9E, 0x3BC21E9D, 0xEF087A76, 0x1D63F975, 0x0E330A81, 0xFC588982 + .word 0xB21572C9, 0x407EF1CA, 0x532E023E, 0xA145813D, 0x758FE5D6, 0x87E466D5, 0x94B49521, 0x66DF1622 + .word 0x38CC2A06, 0xCAA7A905, 0xD9F75AF1, 0x2B9CD9F2, 0xFF56BD19, 0x0D3D3E1A, 0x1E6DCDEE, 0xEC064EED + .word 0xC38D26C4, 0x31E6A5C7, 0x22B65633, 0xD0DDD530, 0x0417B1DB, 0xF67C32D8, 0xE52CC12C, 0x1747422F + .word 0x49547E0B, 0xBB3FFD08, 0xA86F0EFC, 0x5A048DFF, 0x8ECEE914, 0x7CA56A17, 0x6FF599E3, 0x9D9E1AE0 + .word 0xD3D3E1AB, 0x21B862A8, 0x32E8915C, 0xC083125F, 0x144976B4, 0xE622F5B7, 0xF5720643, 0x07198540 + .word 0x590AB964, 0xAB613A67, 0xB831C993, 0x4A5A4A90, 0x9E902E7B, 0x6CFBAD78, 0x7FAB5E8C, 0x8DC0DD8F + .word 0xE330A81A, 0x115B2B19, 0x020BD8ED, 0xF0605BEE, 0x24AA3F05, 0xD6C1BC06, 0xC5914FF2, 0x37FACCF1 + .word 0x69E9F0D5, 0x9B8273D6, 0x88D28022, 0x7AB90321, 0xAE7367CA, 0x5C18E4C9, 0x4F48173D, 0xBD23943E + .word 0xF36E6F75, 0x0105EC76, 0x12551F82, 0xE03E9C81, 0x34F4F86A, 0xC69F7B69, 0xD5CF889D, 0x27A40B9E + .word 0x79B737BA, 0x8BDCB4B9, 0x988C474D, 0x6AE7C44E, 0xBE2DA0A5, 0x4C4623A6, 0x5F16D052, 0xAD7D5351 diff --git a/src/isa-l/crc/aarch64/crc32_mix_default.S b/src/isa-l/crc/aarch64/crc32_mix_default.S new file mode 100644 index 000000000..05c34074d --- /dev/null +++ b/src/isa-l/crc/aarch64/crc32_mix_default.S @@ -0,0 +1,107 @@ +/********************************************************************** + Copyright(c) 2020 Arm Corporation All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Arm Corporation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +**********************************************************************/ + + .arch armv8-a+crypto+crc + .text + .align 6 + +#define CRC32 + +.macro crc32_u64 dst,src,data + crc32x \dst,\src,\data +.endm + +.macro crc32_u32 dst,src,data + crc32w \dst,\src,\data +.endm + +.macro crc32_u16 dst,src,data + crc32h \dst,\src,\data +.endm + +.macro crc32_u8 dst,src,data + crc32b \dst,\src,\data +.endm + +#include "crc32_mix_default_common.S" + + .global crc32_mix_default + .type crc32_mix_default, %function +crc32_mix_default: + crc32_mix_main_default + .size crc32_mix_default, .-crc32_mix_default + + .section .rodata + .align 4 + .set lanchor_crc32,. + 0 + + .type k1k2, %object + .size k1k2, 16 +k1k2: + .xword 0x0154442bd4 + .xword 0x01c6e41596 + + .type k3k4, %object + .size k3k4, 16 +k3k4: + .xword 0x01751997d0 + .xword 0x00ccaa009e + + .type k5k0, %object + .size k5k0, 16 +k5k0: + .xword 0x0163cd6124 + .xword 0 + + .type poly, %object + .size poly, 16 +poly: + .xword 0x01db710641 + .xword 0x01f7011641 + + .type crc32_const, %object + .size crc32_const, 48 +crc32_const: + .xword 0x1753ab84 + .xword 0 + .xword 0xbbf2f6d6 + .xword 0 + .xword 0x0c30f51d + .xword 0 + + .align 4 + .set .lanchor_mask,. + 0 + + .type mask, %object + .size mask, 16 +mask: + .word -1 + .word 0 + .word -1 + .word 0 diff --git a/src/isa-l/crc/aarch64/crc32_mix_default_common.S b/src/isa-l/crc/aarch64/crc32_mix_default_common.S new file mode 100644 index 000000000..106da209a --- /dev/null +++ b/src/isa-l/crc/aarch64/crc32_mix_default_common.S @@ -0,0 +1,563 @@ +/********************************************************************** + Copyright(c) 2020 Arm Corporation All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Arm Corporation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +**********************************************************************/ + +.macro declare_generic_reg name:req, reg:req, default:req + \name .req \default\reg + w_\name .req w\reg + x_\name .req x\reg +.endm + +.macro declare_neon_reg name:req, reg:req, default:req + \name .req \default\reg + v_\name .req v\reg + q_\name .req q\reg + d_\name .req d\reg + s_\name .req s\reg +.endm + +/********************************************************************** + variables +**********************************************************************/ + declare_generic_reg crc, 0,w + declare_generic_reg buf, 1,x + declare_generic_reg len, 2,x + declare_generic_reg buf_saved, 3,x + declare_generic_reg buf_iter, 4,x + declare_generic_reg len_saved, 5,x + declare_generic_reg buf_tmp, 6,x + + declare_generic_reg crc0, 7,x + declare_generic_reg crc1, 8,x + declare_generic_reg crc2, 9,x + declare_generic_reg pconst, 10,x + declare_generic_reg data_crc0, 11,x + declare_generic_reg data_crc1, 12,x + declare_generic_reg data_crc2, 13,x + + declare_generic_reg size, 9,x + declare_generic_reg crc_tmp, 10,w + declare_generic_reg size_tmp, 11,x + declare_generic_reg data_tmp1, 11,x + declare_generic_reg data_tmp2, 12,x + declare_generic_reg data_tmp3, 13,x + + declare_generic_reg tmp, 14,x + declare_generic_reg tmp1, 15,x + +// return + declare_generic_reg ret_crc, 0,w + +/********************************************************************** + simd variables +**********************************************************************/ + declare_neon_reg a0, 0,v + declare_neon_reg a1, 1,v + declare_neon_reg a2, 2,v + declare_neon_reg a3, 3,v + declare_neon_reg a4, 4,v + + declare_neon_reg a5, 16,v + declare_neon_reg a6, 17,v + declare_neon_reg a7, 18,v + declare_neon_reg a8, 19,v + + declare_neon_reg y5, 20,v + declare_neon_reg y6, 21,v + declare_neon_reg y7, 22,v + declare_neon_reg y8, 23,v + + declare_neon_reg neon_zero, 24,v + declare_neon_reg neon_tmp, 24,v + + declare_neon_reg k5k0, 25,v + declare_neon_reg neon_tmp1, 26,v + declare_neon_reg neon_tmp2, 27,v + declare_neon_reg neon_tmp3, 28,v + + declare_neon_reg crc_pmull, 29,v + declare_neon_reg neon_crc0, 30,v + declare_neon_reg neon_crc1, 31,v + + declare_neon_reg neon_const0, 5,v + declare_neon_reg neon_const1, 6,v + declare_neon_reg neon_const2, 7,v + +// constants + .equ offset_k3k4, 16 + .equ offset_k5k0, 32 + .equ offset_poly, 48 + .equ offset_crc32_const, 64 + +// pmull fold +.macro pmull_fold + ldr x_data_crc0, [x_buf_tmp, 464] + ldr x_data_crc1, [x_buf_tmp, 976] + ldr x_data_crc2, [x_buf_tmp, 1488] + + pmull v_a5.1q, v_a1.1d, v_a0.1d + crc32_u64 w_crc0, w_crc0, x_data_crc0 + crc32_u64 w_crc1, w_crc1, x_data_crc1 + crc32_u64 w_crc2, w_crc2, x_data_crc2 + + ldr x_data_crc0, [x_buf_tmp, 472] + ldr x_data_crc1, [x_buf_tmp, 984] + ldr x_data_crc2, [x_buf_tmp, 1496] + + pmull v_a6.1q, v_a2.1d, v_a0.1d + crc32_u64 w_crc0, w_crc0, x_data_crc0 + crc32_u64 w_crc1, w_crc1, x_data_crc1 + crc32_u64 w_crc2, w_crc2, x_data_crc2 + + ldr x_data_crc0, [x_buf_tmp, 480] + ldr x_data_crc1, [x_buf_tmp, 992] + ldr x_data_crc2, [x_buf_tmp, 1504] + + pmull v_a7.1q, v_a3.1d, v_a0.1d + crc32_u64 w_crc0, w_crc0, x_data_crc0 + crc32_u64 w_crc1, w_crc1, x_data_crc1 + crc32_u64 w_crc2, w_crc2, x_data_crc2 + + ldr x_data_crc0, [x_buf_tmp, 488] + ldr x_data_crc1, [x_buf_tmp, 1000] + ldr x_data_crc2, [x_buf_tmp, 1512] + + pmull v_a8.1q, v_a4.1d, v_a0.1d + crc32_u64 w_crc0, w_crc0, x_data_crc0 + crc32_u64 w_crc1, w_crc1, x_data_crc1 + crc32_u64 w_crc2, w_crc2, x_data_crc2 + + ldr x_data_crc0, [x_buf_tmp, 496] + ldr x_data_crc1, [x_buf_tmp, 1008] + ldr x_data_crc2, [x_buf_tmp, 1520] + + pmull2 v_a1.1q, v_a1.2d, v_a0.2d + crc32_u64 w_crc0, w_crc0, x_data_crc0 + crc32_u64 w_crc1, w_crc1, x_data_crc1 + crc32_u64 w_crc2, w_crc2, x_data_crc2 + + ld1 {v_y5.4s, v_y6.4s, v_y7.4s, v_y8.4s}, [x_buf_tmp] + + ldr x_data_crc0, [x_buf_tmp, 504] + ldr x_data_crc1, [x_buf_tmp, 1016] + ldr x_data_crc2, [x_buf_tmp, 1528] + + pmull2 v_a2.1q, v_a2.2d, v_a0.2d + crc32_u64 w_crc0, w_crc0, x_data_crc0 + crc32_u64 w_crc1, w_crc1, x_data_crc1 + crc32_u64 w_crc2, w_crc2, x_data_crc2 + + pmull2 v_a3.1q, v_a3.2d, v_a0.2d + pmull2 v_a4.1q, v_a4.2d, v_a0.2d + + eor v_y5.16b, v_y5.16b, v_a5.16b + eor v_y6.16b, v_y6.16b, v_a6.16b + eor v_y7.16b, v_y7.16b, v_a7.16b + eor v_y8.16b, v_y8.16b, v_a8.16b + + ldr x_data_crc0, [x_buf_tmp, 512] + ldr x_data_crc1, [x_buf_tmp, 1024] + ldr x_data_crc2, [x_buf_tmp, 1536] + + eor v_a1.16b, v_y5.16b, v_a1.16b + eor v_a2.16b, v_y6.16b, v_a2.16b + eor v_a3.16b, v_y7.16b, v_a3.16b + eor v_a4.16b, v_y8.16b, v_a4.16b + + crc32_u64 w_crc0, w_crc0, x_data_crc0 + crc32_u64 w_crc1, w_crc1, x_data_crc1 + crc32_u64 w_crc2, w_crc2, x_data_crc2 + + ldr x_data_crc0, [x_buf_tmp, 520] + ldr x_data_crc1, [x_buf_tmp, 1032] + ldr x_data_crc2, [x_buf_tmp, 1544] + + crc32_u64 w_crc0, w_crc0, x_data_crc0 + crc32_u64 w_crc1, w_crc1, x_data_crc1 + crc32_u64 w_crc2, w_crc2, x_data_crc2 +.endm + +// crc32 mix for 2048 byte input data +.macro crc32_mix2048 + fmov s_a1, w_crc + movi v_neon_tmp.4s, 0 + + adrp x_pconst, lanchor_crc32 + add x_buf_tmp, x_buf, 64 + + ldr x_data_crc0, [x_buf, 512] + ldr x_data_crc1, [x_buf, 1024] + ldr x_data_crc2, [x_buf, 1536] + + crc32_u64 w_crc0, wzr, x_data_crc0 + crc32_u64 w_crc1, wzr, x_data_crc1 + crc32_u64 w_crc2, wzr, x_data_crc2 + +#ifdef CRC32 + mvn v_a1.8b, v_a1.8b +#endif + + ins v_neon_tmp.s[0], v_a1.s[0] + + ld1 {v_a1.4s, v_a2.4s, v_a3.4s, v_a4.4s}, [x_buf] + + ldr x_data_crc0, [x_buf, 520] + ldr x_data_crc1, [x_buf, 1032] + ldr x_data_crc2, [x_buf, 1544] + + eor v_a1.16b, v_a1.16b, v_neon_tmp.16b + ldr q_a0, [x_pconst, #:lo12:lanchor_crc32] // k1k2 + + crc32_u64 w_crc0, w_crc0, x_data_crc0 + crc32_u64 w_crc1, w_crc1, x_data_crc1 + crc32_u64 w_crc2, w_crc2, x_data_crc2 + +// loop start, unroll the loop + .align 4 + pmull_fold + + add x_buf_tmp, x_buf_tmp, 64 + pmull_fold + + add x_buf_tmp, x_buf_tmp, 64 + pmull_fold + + add x_buf_tmp, x_buf_tmp, 64 + pmull_fold + + add x_buf_tmp, x_buf_tmp, 64 + pmull_fold + + add x_buf_tmp, x_buf_tmp, 64 + pmull_fold + + add x_buf_tmp, x_buf_tmp, 64 + pmull_fold +// loop end + +// PMULL: fold into 128-bits + add x_pconst, x_pconst, :lo12:lanchor_crc32 + + ldr x_data_crc0, [x_buf, 976] + ldr x_data_crc1, [x_buf, 1488] + ldr x_data_crc2, [x_buf, 2000] + + ldr q_a0, [x_pconst, offset_k3k4] // k3k4 + + crc32_u64 w_crc0, w_crc0, x_data_crc0 + crc32_u64 w_crc1, w_crc1, x_data_crc1 + crc32_u64 w_crc2, w_crc2, x_data_crc2 + + pmull v_a5.1q, v_a1.1d, v_a0.1d + pmull2 v_a1.1q, v_a1.2d, v_a0.2d + + eor v_a1.16b, v_a5.16b, v_a1.16b + eor v_a1.16b, v_a1.16b, v_a2.16b + + ldr x_data_crc0, [x_buf, 984] + ldr x_data_crc1, [x_buf, 1496] + ldr x_data_crc2, [x_buf, 2008] + + crc32_u64 w_crc0, w_crc0, x_data_crc0 + crc32_u64 w_crc1, w_crc1, x_data_crc1 + crc32_u64 w_crc2, w_crc2, x_data_crc2 + + pmull v_a5.1q, v_a1.1d, v_a0.1d + pmull2 v_a1.1q, v_a1.2d, v_a0.2d + + ldr x_data_crc0, [x_buf, 992] + ldr x_data_crc1, [x_buf, 1504] + ldr x_data_crc2, [x_buf, 2016] + + eor v_a1.16b, v_a5.16b, v_a1.16b + eor v_a1.16b, v_a1.16b, v_a3.16b + + crc32_u64 w_crc0, w_crc0, x_data_crc0 + crc32_u64 w_crc1, w_crc1, x_data_crc1 + crc32_u64 w_crc2, w_crc2, x_data_crc2 + + pmull v_a5.1q, v_a1.1d, v_a0.1d + pmull2 v_a1.1q, v_a1.2d, v_a0.2d + + ldr x_data_crc0, [x_buf, 1000] + ldr x_data_crc1, [x_buf, 1512] + ldr x_data_crc2, [x_buf, 2024] + + eor v_a1.16b, v_a5.16b, v_a1.16b + eor v_a1.16b, v_a1.16b, v_a4.16b + +// PMULL: fold 128-bits to 64-bits + crc32_u64 w_crc0, w_crc0, x_data_crc0 + crc32_u64 w_crc1, w_crc1, x_data_crc1 + crc32_u64 w_crc2, w_crc2, x_data_crc2 + + dup d_a0, v_a0.d[1] + pmull v_a2.1q, v_a1.1d, v_a0.1d + + movi v_neon_zero.4s, 0 + ldr q_k5k0, [x_pconst, offset_k5k0] // k5k0 + adrp x_tmp, .lanchor_mask + + ldr x_data_crc0, [x_buf, 1008] + ldr x_data_crc1, [x_buf, 1520] + ldr x_data_crc2, [x_buf, 2032] + + ext v_a1.16b, v_a1.16b, v_neon_zero.16b, #8 + eor v_a1.16b, v_a2.16b, v_a1.16b + ldr q_neon_tmp3, [x_tmp, #:lo12:.lanchor_mask] + + crc32_u64 w_crc0, w_crc0, x_data_crc0 + crc32_u64 w_crc1, w_crc1, x_data_crc1 + crc32_u64 w_crc2, w_crc2, x_data_crc2 + + dup d_a0, v_k5k0.d[1] + pmull v_a3.1q, v_a2.1d, v_a0.1d + + ext v_a2.16b, v_a1.16b, v_neon_zero.16b, #4 + and v_a1.16b, v_a1.16b, v_neon_tmp3.16b + pmull v_a1.1q, v_a1.1d, v_k5k0.1d + eor v_a1.16b, v_a2.16b, v_a1.16b + +// PMULL: barret reduce to 32-bits + ldr q_neon_tmp1, [x_pconst, offset_poly] // poly + + ldr x_data_crc0, [x_buf, 1016] + ldr x_data_crc1, [x_buf, 1528] + ldr x_data_crc2, [x_buf, 2040] + + dup d_neon_tmp2, v_neon_tmp1.d[1] + + crc32_u64 w_crc0, w_crc0, x_data_crc0 + crc32_u64 w_crc1, w_crc1, x_data_crc1 + crc32_u64 w_crc2, w_crc2, x_data_crc2 + + and v_a2.16b, v_a1.16b, v_neon_tmp3.16b + pmull v_a2.1q, v_a2.1d, v_neon_tmp2.1d + and v_a2.16b, v_neon_tmp3.16b, v_a2.16b + pmull v_a2.1q, v_a2.1d, v_neon_tmp1.1d + +// crc_pmull result + eor v_a1.16b, v_a1.16b, v_a2.16b + dup s_crc_pmull, v_a1.s[1] + +// merge crc_pmull, crc0, crc1, crc2 using pmull instruction + fmov s_neon_crc0, w_crc0 + fmov s_neon_crc1, w_crc1 + + ldr q_neon_const0, [x_pconst, offset_crc32_const] + ldr q_neon_const1, [x_pconst, offset_crc32_const+16] + ldr q_neon_const2, [x_pconst, offset_crc32_const+32] + + pmull v_crc_pmull.1q, v_crc_pmull.1d, v_neon_const0.1d + pmull v_neon_crc0.1q, v_neon_crc0.1d, v_neon_const1.1d + pmull v_neon_crc1.1q, v_neon_crc1.1d, v_neon_const2.1d + + fmov x_tmp1, d_neon_crc0 + crc32_u64 w_crc0, wzr, x_tmp1 + + fmov x_tmp1, d_neon_crc1 + crc32_u64 w_crc1, wzr, x_tmp1 + + eor w_ret_crc, w_crc1, w_crc0 + + fmov x_tmp1, d_crc_pmull + crc32_u64 w_tmp, wzr, x_tmp1 + + eor w_crc2, w_tmp, w_crc2 + +// handle crc32/crc32c +#ifdef CRC32 + eon w_ret_crc, w_crc2, w_ret_crc +#else + eor w_ret_crc, w_crc2, w_ret_crc +#endif +.endm + +// crc32 mix main default +.macro crc32_mix_main_default + cmp x_len, 2047 + mov x_len_saved, x_len + mov x_buf_saved, x_buf + bls .less_than_2048 + + sub x_buf_iter, x_len, #2048 + stp x29, x30, [sp, -16]! + + mov x29, sp + and x_buf_iter, x_buf_iter, -2048 + add x_buf_iter, x_buf_iter, 2048 + add x_buf_iter, x_buf, x_buf_iter + + .align 4 +.loop_mix: + mov x_buf, x_buf_saved + crc32_mix2048 + + add x_buf_saved, x_buf_saved, 2048 + cmp x_buf_saved, x_buf_iter + bne .loop_mix + + and x_len_saved, x_len_saved, 2047 + cbnz x_len_saved, .remain_ldp + + ldp x29, x30, [sp], 16 + ret + + .align 4 +.remain_ldp: + mov w_crc_tmp, crc + ldp x29, x30, [sp], 16 + mov size, x_len_saved + mov buf, x_buf_iter + b .crc32_hw_handle + +.remain: + mov w_crc_tmp, crc + mov size, x_len_saved + mov buf, x_buf_saved + b .crc32_hw_handle + + .align 4 +.less_than_2048: + cbnz x_len, .remain + ret + +.crc32_hw_handle: + cmp size, 63 + +#ifdef CRC32 + mvn crc_tmp, crc_tmp +#endif + + bls .less_than_64 + sub buf_saved, size, #64 + and buf_saved, buf_saved, -64 + add buf_saved, buf_saved, 64 + add buf_saved, buf, buf_saved + + .align 4 +.loop_64: + ldp data_tmp1, data_tmp2, [buf] + ldr data_tmp3, [buf, 16] + crc32_u64 crc_tmp, crc_tmp, data_tmp1 + crc32_u64 crc_tmp, crc_tmp, data_tmp2 + + ldp data_tmp1, data_tmp2, [buf, 24] + add buf, buf, 64 + + crc32_u64 crc_tmp, crc_tmp, data_tmp3 + ldr data_tmp3, [buf, -24] + + crc32_u64 crc_tmp, crc_tmp, data_tmp1 + crc32_u64 crc_tmp, crc_tmp, data_tmp2 + + ldp data_tmp1, data_tmp2, [buf, -16] + cmp buf_saved, buf + crc32_u64 crc_tmp, crc_tmp, data_tmp3 + + crc32_u64 crc_tmp, crc_tmp, data_tmp1 + crc32_u64 crc_tmp, crc_tmp, data_tmp2 + bne .loop_64 + + and size, size, 63 +.less_than_64: + cmp size, 7 + bls .crc32_hw_w + + ldr data_tmp2, [buf] + sub size_tmp, size, #8 + cmp size_tmp, 7 + crc32_u64 crc_tmp, crc_tmp, data_tmp2 + bls .crc32_hw_w_pre + + ldr data_tmp2, [buf, 8] + sub data_tmp3, size, #16 + cmp data_tmp3, 7 + crc32_u64 crc_tmp, crc_tmp, data_tmp2 + bls .crc32_hw_w_pre + + ldr data_tmp2, [buf, 16] + sub data_tmp3, size, #24 + cmp data_tmp3, 7 + crc32_u64 crc_tmp, crc_tmp, data_tmp2 + bls .crc32_hw_w_pre + + ldr data_tmp2, [buf, 24] + sub data_tmp3, size, #32 + cmp data_tmp3, 7 + crc32_u64 crc_tmp, crc_tmp, data_tmp2 + bls .crc32_hw_w_pre + + ldr data_tmp2, [buf, 32] + sub data_tmp3, size, #40 + cmp data_tmp3, 7 + crc32_u64 crc_tmp, crc_tmp, data_tmp2 + bls .crc32_hw_w_pre + + ldr data_tmp2, [buf, 40] + sub data_tmp3, size, #48 + cmp data_tmp3, 7 + crc32_u64 crc_tmp, crc_tmp, data_tmp2 + bls .crc32_hw_w_pre + + ldr data_tmp2, [buf, 48] + crc32_u64 crc_tmp, crc_tmp, data_tmp2 + +.crc32_hw_w_pre: + and size_tmp, size_tmp, -8 + and size, size, 7 + add size_tmp, size_tmp, 8 + add buf, buf, size_tmp + +.crc32_hw_w: + cmp size, 3 + bls .crc32_hw_h + ldr w_data_tmp2, [buf], 4 + sub size, size, #4 + crc32_u32 crc_tmp, crc_tmp, w_data_tmp2 + +.crc32_hw_h: + cmp size, 1 + bls .crc32_hw_b + ldrh w_data_tmp2, [buf], 2 + sub size, size, #2 + crc32_u16 crc_tmp, crc_tmp, w_data_tmp2 + +.crc32_hw_b: + cbz size, .crc32_hw_done + ldrb w_data_tmp2, [buf] + crc32_u8 crc_tmp, crc_tmp, w_data_tmp2 + +.crc32_hw_done: +#ifdef CRC32 + mvn ret_crc, crc_tmp +#else + mov ret_crc, crc_tmp +#endif + ret +.endm diff --git a/src/isa-l/crc/aarch64/crc32_mix_neoverse_n1.S b/src/isa-l/crc/aarch64/crc32_mix_neoverse_n1.S new file mode 100644 index 000000000..62b40e1f2 --- /dev/null +++ b/src/isa-l/crc/aarch64/crc32_mix_neoverse_n1.S @@ -0,0 +1,70 @@ +/********************************************************************** + Copyright(c) 2020 Arm Corporation All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Arm Corporation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +**********************************************************************/ + + .text + .align 6 + .arch armv8-a+crypto+crc + +#include "crc32_common_mix_neoverse_n1.S" +.Lconstants: + .octa 0x00000001c6e415960000000154442bd4 + .octa 0x00000000ccaa009e00000001751997d0 + .octa 0x00000001F701164100000001DB710641 + .quad 0x0000000163cd6124 + .quad 0x00000000FFFFFFFF + .quad 0x000000001753ab84 +.macro crc32_u64 dst,src,data + crc32x \dst,\src,\data +.endm +.macro crc32_u32 dst,src,data + crc32w \dst,\src,\data +.endm +.macro crc32_u16 dst,src,data + crc32h \dst,\src,\data +.endm +.macro crc32_u8 dst,src,data + crc32b \dst,\src,\data +.endm + + +/** + * uint32_t crc32_mix_neoverse_n1(uint CRC ,uint8_t * BUF, + * size_t LEN) + */ + BUF .req x1 + LEN .req x2 + CRC .req x0 + wCRC .req w0 + .align 6 + .global crc32_mix_neoverse_n1 + .type crc32_mix_neoverse_n1, %function +crc32_mix_neoverse_n1: + crc32_common_mix crc32 + .size crc32_mix_neoverse_n1, .-crc32_mix_neoverse_n1 + diff --git a/src/isa-l/crc/aarch64/crc32_norm_common_pmull.h b/src/isa-l/crc/aarch64/crc32_norm_common_pmull.h new file mode 100644 index 000000000..7377e30a1 --- /dev/null +++ b/src/isa-l/crc/aarch64/crc32_norm_common_pmull.h @@ -0,0 +1,135 @@ +######################################################################## +# Copyright(c) 2019 Arm Corporation All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Arm Corporation nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +######################################################################### + +#include "crc_common_pmull.h" + +.macro crc32_norm_func name:req + .arch armv8-a+crypto + .text + .align 3 + .global \name + .type \name, %function + +/* uint32_t crc32_norm_func(uint32_t seed, uint8_t * buf, uint64_t len) */ + +\name\(): + mvn w_seed, w_seed + mov x_counter, 0 + cmp x_len, (FOLD_SIZE - 1) + bhi .crc_clmul_pre + +.crc_tab_pre: + cmp x_len, x_counter + bls .done + + adrp x_tmp, .lanchor_crc_tab + add x_buf_iter, x_buf, x_counter + add x_buf, x_buf, x_len + add x_crc_tab_addr, x_tmp, :lo12:.lanchor_crc_tab + + .align 3 +.loop_crc_tab: + ldrb w_tmp, [x_buf_iter], 1 + cmp x_buf, x_buf_iter + eor w_tmp, w_tmp, w_seed, lsr 24 + ldr w_tmp, [x_crc_tab_addr, w_tmp, uxtw 2] + eor w_seed, w_tmp, w_seed, lsl 8 + bhi .loop_crc_tab + +.done: + mvn w_crc_ret, w_seed + ret + + .align 2 +.crc_clmul_pre: + lsl x_seed, x_seed, 32 + movi v_x0.2s, 0 + fmov v_x0.d[1], x_seed // save crc to v_x0 + + crc_norm_load_first_block + + bls .clmul_loop_end + + crc32_load_p4 + +// 1024bit --> 512bit loop +// merge x0, x1, x2, x3, y0, y1, y2, y3 => x0, x1, x2, x3 (uint64x2_t) + crc_norm_loop + +.clmul_loop_end: +// folding 512bit --> 128bit + crc32_fold_512b_to_128b + +// folding 128bit --> 64bit + mov x_tmp, p0_high_b0 + movk x_tmp, p0_high_b1, lsl 16 + fmov d_p0_high, x_tmp + + mov x_tmp2, p0_low_b0 + movk x_tmp2, p0_low_b1, lsl 16 + fmov d_p0_high2, x_tmp2 + + mov d_tmp_high, v_x3.d[0] + ext v_tmp_high.16b, v_tmp_high.16b, v_tmp_high.16b, #12 + + pmull2 v_x3.1q, v_x3.2d, v_p0.2d + + eor v_tmp_high.16b, v_tmp_high.16b, v_x3.16b + pmull2 v_x3.1q, v_tmp_high.2d, v_p02.2d + +// barrett reduction + mov x_tmp2, br_high_b0 + movk x_tmp2, br_high_b1, lsl 16 + movk x_tmp2, br_high_b2, lsl 32 + fmov d_br_high, x_tmp2 + + mov x_tmp, br_low_b0 + movk x_tmp, br_low_b1, lsl 16 + movk x_tmp, br_low_b2, lsl 32 + fmov d_br_low, x_tmp + + eor v_tmp_high.16b, v_tmp_high.16b, v_x3.16b + mov s_x3, v_tmp_high.s[1] + pmull v_x3.1q, v_x3.1d, v_br_low.1d + + mov s_x3, v_x3.s[1] + pmull v_x3.1q, v_x3.1d, v_br_high.1d + eor v_tmp_high.8b, v_tmp_high.8b, v_x3.8b + umov w_seed, v_tmp_high.s[0] + + b .crc_tab_pre + + .size \name, .-\name + + .section .rodata.cst16,"aM",@progbits,16 + .align 4 +.shuffle_data: + .byte 15, 14, 13, 12, 11, 10, 9 + .byte 8, 7, 6, 5, 4, 3, 2, 1, 0 +.endm diff --git a/src/isa-l/crc/aarch64/crc32_refl_common_pmull.h b/src/isa-l/crc/aarch64/crc32_refl_common_pmull.h new file mode 100644 index 000000000..6418f1240 --- /dev/null +++ b/src/isa-l/crc/aarch64/crc32_refl_common_pmull.h @@ -0,0 +1,126 @@ +######################################################################## +# Copyright(c) 2019 Arm Corporation All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Arm Corporation nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +######################################################################### + +#include "crc_common_pmull.h" + +.macro crc32_refl_func name:req + .arch armv8-a+crypto + .text + .align 3 + .global \name + .type \name, %function + +/* uint32_t crc32_refl_func(uint32_t seed, uint8_t * buf, uint64_t len) */ + +\name\(): + mvn w_seed, w_seed + mov x_counter, 0 + cmp x_len, (FOLD_SIZE - 1) + bhi .crc32_clmul_pre + +.crc_tab_pre: + cmp x_len, x_counter + bls .done + + adrp x_tmp, .lanchor_crc_tab + add x_buf_iter, x_buf, x_counter + add x_buf, x_buf, x_len + add x_crc_tab_addr, x_tmp, :lo12:.lanchor_crc_tab + + .align 3 +.loop_crc_tab: + ldrb w_tmp, [x_buf_iter], 1 + cmp x_buf, x_buf_iter + eor w_tmp, w_tmp, w_seed + and w_tmp, w_tmp, 255 + ldr w_tmp, [x_crc_tab_addr, w_tmp, uxtw 2] + eor w_seed, w_tmp, w_seed, lsr 8 + bhi .loop_crc_tab + +.done: + mvn w_crc_ret, w_seed + ret + + .align 2 +.crc32_clmul_pre: + fmov s_x0, w_seed // save crc to s_x0 + + crc_refl_load_first_block + + bls .clmul_loop_end + + crc32_load_p4 + +// 1024bit --> 512bit loop +// merge x0, x1, x2, x3, y0, y1, y2, y3 => x0, x1, x2, x3 (uint64x2_t) + crc_refl_loop + +.clmul_loop_end: +// folding 512bit --> 128bit + crc32_fold_512b_to_128b + +// folding 128bit --> 64bit + mov x_tmp, p0_low_b0 + movk x_tmp, p0_low_b1, lsl 16 + fmov d_p0_low2, x_tmp + + mov d_tmp_high, v_x3.d[1] + + mov d_p0_low, v_p1.d[1] + pmull v_x3.1q, v_x3.1d, v_p0.1d + + eor v_tmp_high.16b, v_tmp_high.16b, v_x3.16b + mov s_x3, v_tmp_high.s[0] + ext v_tmp_high.16b, v_tmp_high.16b, v_tmp_high.16b, #4 + pmull v_x3.1q, v_x3.1d, v_p02.1d + +// barrett reduction + mov x_tmp2, br_high_b0 + movk x_tmp2, br_high_b1, lsl 16 + movk x_tmp2, br_high_b2, lsl 32 + fmov d_br_high, x_tmp2 + + mov x_tmp, br_low_b0 + movk x_tmp, br_low_b1, lsl 16 + movk x_tmp, br_low_b2, lsl 32 + fmov d_br_low, x_tmp + + eor v_tmp_high.16b, v_tmp_high.16b, v_x3.16b + mov s_x3, v_tmp_high.s[0] + pmull v_x3.1q, v_x3.1d, v_br_high.1d + + mov s_x3, v_x3.s[0] + pmull v_x3.1q, v_x3.1d, v_br_low.1d + eor v_tmp_high.8b, v_tmp_high.8b, v_x3.8b + umov w_seed, v_tmp_high.s[1] + + b .crc_tab_pre + + .size \name, .-\name +.endm diff --git a/src/isa-l/crc/aarch64/crc32c_mix_default.S b/src/isa-l/crc/aarch64/crc32c_mix_default.S new file mode 100644 index 000000000..87b8ce39c --- /dev/null +++ b/src/isa-l/crc/aarch64/crc32c_mix_default.S @@ -0,0 +1,109 @@ +/********************************************************************** + Copyright(c) 2020 Arm Corporation All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Arm Corporation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +**********************************************************************/ + + .text + .arch armv8-a+crypto+crc + .align 6 + +.macro crc32_u64 dst,src,data + crc32cx \dst,\src,\data +.endm + +.macro crc32_u32 dst,src,data + crc32cw \dst,\src,\data +.endm + +.macro crc32_u16 dst,src,data + crc32ch \dst,\src,\data +.endm + +.macro crc32_u8 dst,src,data + crc32cb \dst,\src,\data +.endm + +#include "crc32_mix_default_common.S" + + .global crc32c_mix_default + .type crc32c_mix_default, %function +crc32c_mix_default: + mov w3, w2 + sxtw x2, w1 + mov x1, x0 + mov w0, w3 + crc32_mix_main_default + .size crc32c_mix_default, .-crc32c_mix_default + + .section .rodata + .align 4 + .set lanchor_crc32,. + 0 + + .type k1k2, %object + .size k1k2, 16 +k1k2: + .xword 0x00740eef02 + .xword 0x009e4addf8 + + .type k3k4, %object + .size k3k4, 16 +k3k4: + .xword 0x00f20c0dfe + .xword 0x014cd00bd6 + + .type k5k0, %object + .size k5k0, 16 +k5k0: + .xword 0x00dd45aab8 + .xword 0 + + .type poly, %object + .size poly, 16 +poly: + .xword 0x0105ec76f0 + .xword 0x00dea713f1 + + .type crc32_const, %object + .size crc32_const, 48 +crc32_const: + .xword 0x9ef68d35 + .xword 0 + .xword 0x170076fa + .xword 0 + .xword 0xdd7e3b0c + .xword 0 + + .align 4 + .set .lanchor_mask,. + 0 + + .type mask, %object + .size mask, 16 +mask: + .word -1 + .word 0 + .word -1 + .word 0 diff --git a/src/isa-l/crc/aarch64/crc32c_mix_neoverse_n1.S b/src/isa-l/crc/aarch64/crc32c_mix_neoverse_n1.S new file mode 100644 index 000000000..a98511aab --- /dev/null +++ b/src/isa-l/crc/aarch64/crc32c_mix_neoverse_n1.S @@ -0,0 +1,68 @@ +/********************************************************************** + Copyright(c) 2020 Arm Corporation All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Arm Corporation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +**********************************************************************/ + + .text + .align 6 + .arch armv8-a+crypto+crc + +#include "crc32_common_mix_neoverse_n1.S" +.Lconstants: + .octa 0x000000009e4addf800000000740eef02 + .octa 0x000000014cd00bd600000000f20c0dfe + .octa 0x00000000dea713f10000000105ec76f0 + .quad 0x00000000dd45aab8 + .quad 0x00000000FFFFFFFF + .quad 0x000000009ef68d35 + +.macro crc32_u64 dst,src,data + crc32cx \dst,\src,\data +.endm +.macro crc32_u32 dst,src,data + crc32cw \dst,\src,\data +.endm +.macro crc32_u16 dst,src,data + crc32ch \dst,\src,\data +.endm +.macro crc32_u8 dst,src,data + crc32cb \dst,\src,\data +.endm +/** + * uint32_t crc32c_mix_neoverse_n1(uint8_t * BUF, + * size_t LEN, uint CRC) + */ + BUF .req x0 + LEN .req x1 + CRC .req x2 + wCRC .req w2 + .align 6 + .global crc32c_mix_neoverse_n1 + .type crc32c_mix_neoverse_n1, %function +crc32c_mix_neoverse_n1: + crc32_common_mix crc32c + .size crc32c_mix_neoverse_n1, .-crc32c_mix_neoverse_n1 diff --git a/src/isa-l/crc/aarch64/crc64_ecma_norm_pmull.S b/src/isa-l/crc/aarch64/crc64_ecma_norm_pmull.S new file mode 100644 index 000000000..0089a09de --- /dev/null +++ b/src/isa-l/crc/aarch64/crc64_ecma_norm_pmull.S @@ -0,0 +1,33 @@ +######################################################################## +# Copyright(c) 2019 Arm Corporation All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Arm Corporation nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +######################################################################### + +#include "crc64_ecma_norm_pmull.h" +#include "crc64_norm_common_pmull.h" + +crc64_norm_func crc64_ecma_norm_pmull diff --git a/src/isa-l/crc/aarch64/crc64_ecma_norm_pmull.h b/src/isa-l/crc/aarch64/crc64_ecma_norm_pmull.h new file mode 100644 index 000000000..07d58cd87 --- /dev/null +++ b/src/isa-l/crc/aarch64/crc64_ecma_norm_pmull.h @@ -0,0 +1,200 @@ +######################################################################## +# Copyright(c) 2019 Arm Corporation All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Arm Corporation nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +######################################################################### + +.equ p4_low_b0, (0xf020) +.equ p4_low_b1, 0x540d +.equ p4_low_b2, 0x43ca +.equ p4_low_b3, 0x5f68 +.equ p4_high_b0, 0xb83f +.equ p4_high_b1, 0x1205 +.equ p4_high_b2, 0xb698 +.equ p4_high_b3, 0xddf4 + +.equ p1_low_b0, (0xfab6) +.equ p1_low_b1, 0xeb52 +.equ p1_low_b2, 0xc3c7 +.equ p1_low_b3, 0x05f5 +.equ p1_high_b0, 0x740e +.equ p1_high_b1, 0xd257 +.equ p1_high_b2, 0x38a7 +.equ p1_high_b3, 0x4eb9 + +.equ p0_low_b0, (0xfab6) +.equ p0_low_b1, 0xeb52 +.equ p0_low_b2, 0xc3c7 +.equ p0_low_b3, 0x05f5 +.equ p0_high_b0, 0x0 +.equ p0_high_b1, 0x0 +.equ p0_high_b2, 0x0 +.equ p0_high_b3, 0x0 + +.equ br_low_b0, (0xf872) +.equ br_low_b1, 0x6cc4 +.equ br_low_b2, 0x29d0 +.equ br_low_b3, 0x578d +.equ br_high_b0, 0x3693 +.equ br_high_b1, 0xa9ea +.equ br_high_b2, 0xe1eb +.equ br_high_b3, 0x42f0 + + .text + .section .rodata + .align 4 + .set .lanchor_crc_tab,. + 0 + .type crc64_tab, %object + .size crc64_tab, 2048 +crc64_tab: + .xword 0x0000000000000000, 0x42f0e1eba9ea3693 + .xword 0x85e1c3d753d46d26, 0xc711223cfa3e5bb5 + .xword 0x493366450e42ecdf, 0x0bc387aea7a8da4c + .xword 0xccd2a5925d9681f9, 0x8e224479f47cb76a + .xword 0x9266cc8a1c85d9be, 0xd0962d61b56fef2d + .xword 0x17870f5d4f51b498, 0x5577eeb6e6bb820b + .xword 0xdb55aacf12c73561, 0x99a54b24bb2d03f2 + .xword 0x5eb4691841135847, 0x1c4488f3e8f96ed4 + .xword 0x663d78ff90e185ef, 0x24cd9914390bb37c + .xword 0xe3dcbb28c335e8c9, 0xa12c5ac36adfde5a + .xword 0x2f0e1eba9ea36930, 0x6dfeff5137495fa3 + .xword 0xaaefdd6dcd770416, 0xe81f3c86649d3285 + .xword 0xf45bb4758c645c51, 0xb6ab559e258e6ac2 + .xword 0x71ba77a2dfb03177, 0x334a9649765a07e4 + .xword 0xbd68d2308226b08e, 0xff9833db2bcc861d + .xword 0x388911e7d1f2dda8, 0x7a79f00c7818eb3b + .xword 0xcc7af1ff21c30bde, 0x8e8a101488293d4d + .xword 0x499b3228721766f8, 0x0b6bd3c3dbfd506b + .xword 0x854997ba2f81e701, 0xc7b97651866bd192 + .xword 0x00a8546d7c558a27, 0x4258b586d5bfbcb4 + .xword 0x5e1c3d753d46d260, 0x1cecdc9e94ace4f3 + .xword 0xdbfdfea26e92bf46, 0x990d1f49c77889d5 + .xword 0x172f5b3033043ebf, 0x55dfbadb9aee082c + .xword 0x92ce98e760d05399, 0xd03e790cc93a650a + .xword 0xaa478900b1228e31, 0xe8b768eb18c8b8a2 + .xword 0x2fa64ad7e2f6e317, 0x6d56ab3c4b1cd584 + .xword 0xe374ef45bf6062ee, 0xa1840eae168a547d + .xword 0x66952c92ecb40fc8, 0x2465cd79455e395b + .xword 0x3821458aada7578f, 0x7ad1a461044d611c + .xword 0xbdc0865dfe733aa9, 0xff3067b657990c3a + .xword 0x711223cfa3e5bb50, 0x33e2c2240a0f8dc3 + .xword 0xf4f3e018f031d676, 0xb60301f359dbe0e5 + .xword 0xda050215ea6c212f, 0x98f5e3fe438617bc + .xword 0x5fe4c1c2b9b84c09, 0x1d14202910527a9a + .xword 0x93366450e42ecdf0, 0xd1c685bb4dc4fb63 + .xword 0x16d7a787b7faa0d6, 0x5427466c1e109645 + .xword 0x4863ce9ff6e9f891, 0x0a932f745f03ce02 + .xword 0xcd820d48a53d95b7, 0x8f72eca30cd7a324 + .xword 0x0150a8daf8ab144e, 0x43a04931514122dd + .xword 0x84b16b0dab7f7968, 0xc6418ae602954ffb + .xword 0xbc387aea7a8da4c0, 0xfec89b01d3679253 + .xword 0x39d9b93d2959c9e6, 0x7b2958d680b3ff75 + .xword 0xf50b1caf74cf481f, 0xb7fbfd44dd257e8c + .xword 0x70eadf78271b2539, 0x321a3e938ef113aa + .xword 0x2e5eb66066087d7e, 0x6cae578bcfe24bed + .xword 0xabbf75b735dc1058, 0xe94f945c9c3626cb + .xword 0x676dd025684a91a1, 0x259d31cec1a0a732 + .xword 0xe28c13f23b9efc87, 0xa07cf2199274ca14 + .xword 0x167ff3eacbaf2af1, 0x548f120162451c62 + .xword 0x939e303d987b47d7, 0xd16ed1d631917144 + .xword 0x5f4c95afc5edc62e, 0x1dbc74446c07f0bd + .xword 0xdaad56789639ab08, 0x985db7933fd39d9b + .xword 0x84193f60d72af34f, 0xc6e9de8b7ec0c5dc + .xword 0x01f8fcb784fe9e69, 0x43081d5c2d14a8fa + .xword 0xcd2a5925d9681f90, 0x8fdab8ce70822903 + .xword 0x48cb9af28abc72b6, 0x0a3b7b1923564425 + .xword 0x70428b155b4eaf1e, 0x32b26afef2a4998d + .xword 0xf5a348c2089ac238, 0xb753a929a170f4ab + .xword 0x3971ed50550c43c1, 0x7b810cbbfce67552 + .xword 0xbc902e8706d82ee7, 0xfe60cf6caf321874 + .xword 0xe224479f47cb76a0, 0xa0d4a674ee214033 + .xword 0x67c58448141f1b86, 0x253565a3bdf52d15 + .xword 0xab1721da49899a7f, 0xe9e7c031e063acec + .xword 0x2ef6e20d1a5df759, 0x6c0603e6b3b7c1ca + .xword 0xf6fae5c07d3274cd, 0xb40a042bd4d8425e + .xword 0x731b26172ee619eb, 0x31ebc7fc870c2f78 + .xword 0xbfc9838573709812, 0xfd39626eda9aae81 + .xword 0x3a28405220a4f534, 0x78d8a1b9894ec3a7 + .xword 0x649c294a61b7ad73, 0x266cc8a1c85d9be0 + .xword 0xe17dea9d3263c055, 0xa38d0b769b89f6c6 + .xword 0x2daf4f0f6ff541ac, 0x6f5faee4c61f773f + .xword 0xa84e8cd83c212c8a, 0xeabe6d3395cb1a19 + .xword 0x90c79d3fedd3f122, 0xd2377cd44439c7b1 + .xword 0x15265ee8be079c04, 0x57d6bf0317edaa97 + .xword 0xd9f4fb7ae3911dfd, 0x9b041a914a7b2b6e + .xword 0x5c1538adb04570db, 0x1ee5d94619af4648 + .xword 0x02a151b5f156289c, 0x4051b05e58bc1e0f + .xword 0x87409262a28245ba, 0xc5b073890b687329 + .xword 0x4b9237f0ff14c443, 0x0962d61b56fef2d0 + .xword 0xce73f427acc0a965, 0x8c8315cc052a9ff6 + .xword 0x3a80143f5cf17f13, 0x7870f5d4f51b4980 + .xword 0xbf61d7e80f251235, 0xfd913603a6cf24a6 + .xword 0x73b3727a52b393cc, 0x31439391fb59a55f + .xword 0xf652b1ad0167feea, 0xb4a25046a88dc879 + .xword 0xa8e6d8b54074a6ad, 0xea16395ee99e903e + .xword 0x2d071b6213a0cb8b, 0x6ff7fa89ba4afd18 + .xword 0xe1d5bef04e364a72, 0xa3255f1be7dc7ce1 + .xword 0x64347d271de22754, 0x26c49cccb40811c7 + .xword 0x5cbd6cc0cc10fafc, 0x1e4d8d2b65facc6f + .xword 0xd95caf179fc497da, 0x9bac4efc362ea149 + .xword 0x158e0a85c2521623, 0x577eeb6e6bb820b0 + .xword 0x906fc95291867b05, 0xd29f28b9386c4d96 + .xword 0xcedba04ad0952342, 0x8c2b41a1797f15d1 + .xword 0x4b3a639d83414e64, 0x09ca82762aab78f7 + .xword 0x87e8c60fded7cf9d, 0xc51827e4773df90e + .xword 0x020905d88d03a2bb, 0x40f9e43324e99428 + .xword 0x2cffe7d5975e55e2, 0x6e0f063e3eb46371 + .xword 0xa91e2402c48a38c4, 0xebeec5e96d600e57 + .xword 0x65cc8190991cb93d, 0x273c607b30f68fae + .xword 0xe02d4247cac8d41b, 0xa2dda3ac6322e288 + .xword 0xbe992b5f8bdb8c5c, 0xfc69cab42231bacf + .xword 0x3b78e888d80fe17a, 0x7988096371e5d7e9 + .xword 0xf7aa4d1a85996083, 0xb55aacf12c735610 + .xword 0x724b8ecdd64d0da5, 0x30bb6f267fa73b36 + .xword 0x4ac29f2a07bfd00d, 0x08327ec1ae55e69e + .xword 0xcf235cfd546bbd2b, 0x8dd3bd16fd818bb8 + .xword 0x03f1f96f09fd3cd2, 0x41011884a0170a41 + .xword 0x86103ab85a2951f4, 0xc4e0db53f3c36767 + .xword 0xd8a453a01b3a09b3, 0x9a54b24bb2d03f20 + .xword 0x5d45907748ee6495, 0x1fb5719ce1045206 + .xword 0x919735e51578e56c, 0xd367d40ebc92d3ff + .xword 0x1476f63246ac884a, 0x568617d9ef46bed9 + .xword 0xe085162ab69d5e3c, 0xa275f7c11f7768af + .xword 0x6564d5fde549331a, 0x279434164ca30589 + .xword 0xa9b6706fb8dfb2e3, 0xeb46918411358470 + .xword 0x2c57b3b8eb0bdfc5, 0x6ea7525342e1e956 + .xword 0x72e3daa0aa188782, 0x30133b4b03f2b111 + .xword 0xf7021977f9cceaa4, 0xb5f2f89c5026dc37 + .xword 0x3bd0bce5a45a6b5d, 0x79205d0e0db05dce + .xword 0xbe317f32f78e067b, 0xfcc19ed95e6430e8 + .xword 0x86b86ed5267cdbd3, 0xc4488f3e8f96ed40 + .xword 0x0359ad0275a8b6f5, 0x41a94ce9dc428066 + .xword 0xcf8b0890283e370c, 0x8d7be97b81d4019f + .xword 0x4a6acb477bea5a2a, 0x089a2aacd2006cb9 + .xword 0x14dea25f3af9026d, 0x562e43b4931334fe + .xword 0x913f6188692d6f4b, 0xd3cf8063c0c759d8 + .xword 0x5dedc41a34bbeeb2, 0x1f1d25f19d51d821 + .xword 0xd80c07cd676f8394, 0x9afce626ce85b507 diff --git a/src/isa-l/crc/aarch64/crc64_ecma_refl_pmull.S b/src/isa-l/crc/aarch64/crc64_ecma_refl_pmull.S new file mode 100644 index 000000000..812517f77 --- /dev/null +++ b/src/isa-l/crc/aarch64/crc64_ecma_refl_pmull.S @@ -0,0 +1,33 @@ +######################################################################## +# Copyright(c) 2019 Arm Corporation All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Arm Corporation nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +######################################################################### + +#include "crc64_ecma_refl_pmull.h" +#include "crc64_refl_common_pmull.h" + +crc64_refl_func crc64_ecma_refl_pmull diff --git a/src/isa-l/crc/aarch64/crc64_ecma_refl_pmull.h b/src/isa-l/crc/aarch64/crc64_ecma_refl_pmull.h new file mode 100644 index 000000000..5f53d7903 --- /dev/null +++ b/src/isa-l/crc/aarch64/crc64_ecma_refl_pmull.h @@ -0,0 +1,196 @@ +######################################################################## +# Copyright(c) 2019 Arm Corporation All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Arm Corporation nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +######################################################################### + +.equ p4_low_b0, 0x41f3 +.equ p4_low_b1, 0x9dd4 +.equ p4_low_b2, 0xefbb +.equ p4_low_b3, 0x6ae3 +.equ p4_high_b0, 0x2df4 +.equ p4_high_b1, 0xa784 +.equ p4_high_b2, 0x6054 +.equ p4_high_b3, 0x081f + +.equ p1_low_b0, 0x3ae4 +.equ p1_low_b1, 0xca39 +.equ p1_low_b2, 0xd497 +.equ p1_low_b3, 0xe05d +.equ p1_high_b0, 0x5f40 +.equ p1_high_b1, 0xc787 +.equ p1_high_b2, 0x95af +.equ p1_high_b3, 0xdabe + +.equ p0_low_b0, 0x5f40 +.equ p0_low_b1, 0xc787 +.equ p0_low_b2, 0x95af +.equ p0_low_b3, 0xdabe + +.equ br_low_b0, 0x63d5 +.equ br_low_b1, 0x1729 +.equ br_low_b2, 0x466c +.equ br_low_b3, 0x9c3e +.equ br_high_b0, 0x1e85 +.equ br_high_b1, 0xaf0e +.equ br_high_b2, 0xaf2b +.equ br_high_b3, 0x92d8 + + .text + .section .rodata + .align 4 + .set .lanchor_crc_tab,. + 0 + .type crc64_tab, %object + .size crc64_tab, 2048 +crc64_tab: + .xword 0x0000000000000000, 0xb32e4cbe03a75f6f + .xword 0xf4843657a840a05b, 0x47aa7ae9abe7ff34 + .xword 0x7bd0c384ff8f5e33, 0xc8fe8f3afc28015c + .xword 0x8f54f5d357cffe68, 0x3c7ab96d5468a107 + .xword 0xf7a18709ff1ebc66, 0x448fcbb7fcb9e309 + .xword 0x0325b15e575e1c3d, 0xb00bfde054f94352 + .xword 0x8c71448d0091e255, 0x3f5f08330336bd3a + .xword 0x78f572daa8d1420e, 0xcbdb3e64ab761d61 + .xword 0x7d9ba13851336649, 0xceb5ed8652943926 + .xword 0x891f976ff973c612, 0x3a31dbd1fad4997d + .xword 0x064b62bcaebc387a, 0xb5652e02ad1b6715 + .xword 0xf2cf54eb06fc9821, 0x41e11855055bc74e + .xword 0x8a3a2631ae2dda2f, 0x39146a8fad8a8540 + .xword 0x7ebe1066066d7a74, 0xcd905cd805ca251b + .xword 0xf1eae5b551a2841c, 0x42c4a90b5205db73 + .xword 0x056ed3e2f9e22447, 0xb6409f5cfa457b28 + .xword 0xfb374270a266cc92, 0x48190ecea1c193fd + .xword 0x0fb374270a266cc9, 0xbc9d3899098133a6 + .xword 0x80e781f45de992a1, 0x33c9cd4a5e4ecdce + .xword 0x7463b7a3f5a932fa, 0xc74dfb1df60e6d95 + .xword 0x0c96c5795d7870f4, 0xbfb889c75edf2f9b + .xword 0xf812f32ef538d0af, 0x4b3cbf90f69f8fc0 + .xword 0x774606fda2f72ec7, 0xc4684a43a15071a8 + .xword 0x83c230aa0ab78e9c, 0x30ec7c140910d1f3 + .xword 0x86ace348f355aadb, 0x3582aff6f0f2f5b4 + .xword 0x7228d51f5b150a80, 0xc10699a158b255ef + .xword 0xfd7c20cc0cdaf4e8, 0x4e526c720f7dab87 + .xword 0x09f8169ba49a54b3, 0xbad65a25a73d0bdc + .xword 0x710d64410c4b16bd, 0xc22328ff0fec49d2 + .xword 0x85895216a40bb6e6, 0x36a71ea8a7ace989 + .xword 0x0adda7c5f3c4488e, 0xb9f3eb7bf06317e1 + .xword 0xfe5991925b84e8d5, 0x4d77dd2c5823b7ba + .xword 0x64b62bcaebc387a1, 0xd7986774e864d8ce + .xword 0x90321d9d438327fa, 0x231c512340247895 + .xword 0x1f66e84e144cd992, 0xac48a4f017eb86fd + .xword 0xebe2de19bc0c79c9, 0x58cc92a7bfab26a6 + .xword 0x9317acc314dd3bc7, 0x2039e07d177a64a8 + .xword 0x67939a94bc9d9b9c, 0xd4bdd62abf3ac4f3 + .xword 0xe8c76f47eb5265f4, 0x5be923f9e8f53a9b + .xword 0x1c4359104312c5af, 0xaf6d15ae40b59ac0 + .xword 0x192d8af2baf0e1e8, 0xaa03c64cb957be87 + .xword 0xeda9bca512b041b3, 0x5e87f01b11171edc + .xword 0x62fd4976457fbfdb, 0xd1d305c846d8e0b4 + .xword 0x96797f21ed3f1f80, 0x2557339fee9840ef + .xword 0xee8c0dfb45ee5d8e, 0x5da24145464902e1 + .xword 0x1a083bacedaefdd5, 0xa9267712ee09a2ba + .xword 0x955cce7fba6103bd, 0x267282c1b9c65cd2 + .xword 0x61d8f8281221a3e6, 0xd2f6b4961186fc89 + .xword 0x9f8169ba49a54b33, 0x2caf25044a02145c + .xword 0x6b055fede1e5eb68, 0xd82b1353e242b407 + .xword 0xe451aa3eb62a1500, 0x577fe680b58d4a6f + .xword 0x10d59c691e6ab55b, 0xa3fbd0d71dcdea34 + .xword 0x6820eeb3b6bbf755, 0xdb0ea20db51ca83a + .xword 0x9ca4d8e41efb570e, 0x2f8a945a1d5c0861 + .xword 0x13f02d374934a966, 0xa0de61894a93f609 + .xword 0xe7741b60e174093d, 0x545a57dee2d35652 + .xword 0xe21ac88218962d7a, 0x5134843c1b317215 + .xword 0x169efed5b0d68d21, 0xa5b0b26bb371d24e + .xword 0x99ca0b06e7197349, 0x2ae447b8e4be2c26 + .xword 0x6d4e3d514f59d312, 0xde6071ef4cfe8c7d + .xword 0x15bb4f8be788911c, 0xa6950335e42fce73 + .xword 0xe13f79dc4fc83147, 0x521135624c6f6e28 + .xword 0x6e6b8c0f1807cf2f, 0xdd45c0b11ba09040 + .xword 0x9aefba58b0476f74, 0x29c1f6e6b3e0301b + .xword 0xc96c5795d7870f42, 0x7a421b2bd420502d + .xword 0x3de861c27fc7af19, 0x8ec62d7c7c60f076 + .xword 0xb2bc941128085171, 0x0192d8af2baf0e1e + .xword 0x4638a2468048f12a, 0xf516eef883efae45 + .xword 0x3ecdd09c2899b324, 0x8de39c222b3eec4b + .xword 0xca49e6cb80d9137f, 0x7967aa75837e4c10 + .xword 0x451d1318d716ed17, 0xf6335fa6d4b1b278 + .xword 0xb199254f7f564d4c, 0x02b769f17cf11223 + .xword 0xb4f7f6ad86b4690b, 0x07d9ba1385133664 + .xword 0x4073c0fa2ef4c950, 0xf35d8c442d53963f + .xword 0xcf273529793b3738, 0x7c0979977a9c6857 + .xword 0x3ba3037ed17b9763, 0x888d4fc0d2dcc80c + .xword 0x435671a479aad56d, 0xf0783d1a7a0d8a02 + .xword 0xb7d247f3d1ea7536, 0x04fc0b4dd24d2a59 + .xword 0x3886b22086258b5e, 0x8ba8fe9e8582d431 + .xword 0xcc0284772e652b05, 0x7f2cc8c92dc2746a + .xword 0x325b15e575e1c3d0, 0x8175595b76469cbf + .xword 0xc6df23b2dda1638b, 0x75f16f0cde063ce4 + .xword 0x498bd6618a6e9de3, 0xfaa59adf89c9c28c + .xword 0xbd0fe036222e3db8, 0x0e21ac88218962d7 + .xword 0xc5fa92ec8aff7fb6, 0x76d4de52895820d9 + .xword 0x317ea4bb22bfdfed, 0x8250e80521188082 + .xword 0xbe2a516875702185, 0x0d041dd676d77eea + .xword 0x4aae673fdd3081de, 0xf9802b81de97deb1 + .xword 0x4fc0b4dd24d2a599, 0xfceef8632775faf6 + .xword 0xbb44828a8c9205c2, 0x086ace348f355aad + .xword 0x34107759db5dfbaa, 0x873e3be7d8faa4c5 + .xword 0xc094410e731d5bf1, 0x73ba0db070ba049e + .xword 0xb86133d4dbcc19ff, 0x0b4f7f6ad86b4690 + .xword 0x4ce50583738cb9a4, 0xffcb493d702be6cb + .xword 0xc3b1f050244347cc, 0x709fbcee27e418a3 + .xword 0x3735c6078c03e797, 0x841b8ab98fa4b8f8 + .xword 0xadda7c5f3c4488e3, 0x1ef430e13fe3d78c + .xword 0x595e4a08940428b8, 0xea7006b697a377d7 + .xword 0xd60abfdbc3cbd6d0, 0x6524f365c06c89bf + .xword 0x228e898c6b8b768b, 0x91a0c532682c29e4 + .xword 0x5a7bfb56c35a3485, 0xe955b7e8c0fd6bea + .xword 0xaeffcd016b1a94de, 0x1dd181bf68bdcbb1 + .xword 0x21ab38d23cd56ab6, 0x9285746c3f7235d9 + .xword 0xd52f0e859495caed, 0x6601423b97329582 + .xword 0xd041dd676d77eeaa, 0x636f91d96ed0b1c5 + .xword 0x24c5eb30c5374ef1, 0x97eba78ec690119e + .xword 0xab911ee392f8b099, 0x18bf525d915feff6 + .xword 0x5f1528b43ab810c2, 0xec3b640a391f4fad + .xword 0x27e05a6e926952cc, 0x94ce16d091ce0da3 + .xword 0xd3646c393a29f297, 0x604a2087398eadf8 + .xword 0x5c3099ea6de60cff, 0xef1ed5546e415390 + .xword 0xa8b4afbdc5a6aca4, 0x1b9ae303c601f3cb + .xword 0x56ed3e2f9e224471, 0xe5c372919d851b1e + .xword 0xa26908783662e42a, 0x114744c635c5bb45 + .xword 0x2d3dfdab61ad1a42, 0x9e13b115620a452d + .xword 0xd9b9cbfcc9edba19, 0x6a978742ca4ae576 + .xword 0xa14cb926613cf817, 0x1262f598629ba778 + .xword 0x55c88f71c97c584c, 0xe6e6c3cfcadb0723 + .xword 0xda9c7aa29eb3a624, 0x69b2361c9d14f94b + .xword 0x2e184cf536f3067f, 0x9d36004b35545910 + .xword 0x2b769f17cf112238, 0x9858d3a9ccb67d57 + .xword 0xdff2a94067518263, 0x6cdce5fe64f6dd0c + .xword 0x50a65c93309e7c0b, 0xe388102d33392364 + .xword 0xa4226ac498dedc50, 0x170c267a9b79833f + .xword 0xdcd7181e300f9e5e, 0x6ff954a033a8c131 + .xword 0x28532e49984f3e05, 0x9b7d62f79be8616a + .xword 0xa707db9acf80c06d, 0x14299724cc279f02 + .xword 0x5383edcd67c06036, 0xe0ada17364673f59 diff --git a/src/isa-l/crc/aarch64/crc64_iso_norm_pmull.S b/src/isa-l/crc/aarch64/crc64_iso_norm_pmull.S new file mode 100644 index 000000000..185b75bdf --- /dev/null +++ b/src/isa-l/crc/aarch64/crc64_iso_norm_pmull.S @@ -0,0 +1,33 @@ +######################################################################## +# Copyright(c) 2019 Arm Corporation All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Arm Corporation nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +######################################################################### + +#include "crc64_iso_norm_pmull.h" +#include "crc64_norm_common_pmull.h" + +crc64_norm_func crc64_iso_norm_pmull diff --git a/src/isa-l/crc/aarch64/crc64_iso_norm_pmull.h b/src/isa-l/crc/aarch64/crc64_iso_norm_pmull.h new file mode 100644 index 000000000..cc176051c --- /dev/null +++ b/src/isa-l/crc/aarch64/crc64_iso_norm_pmull.h @@ -0,0 +1,201 @@ +######################################################################## +# Copyright(c) 2019 Arm Corporation All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Arm Corporation nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +######################################################################### + +.equ p4_low_b0, (0x0101) +.equ p4_low_b1, 0x0100 +.equ p4_low_b2, 0x0001 +.equ p4_low_b3, 0x0000 +.equ p4_high_b0, 0x1b1b +.equ p4_high_b1, 0x1b00 +.equ p4_high_b2, 0x001b +.equ p4_high_b3, 0x0000 + +.equ p1_low_b0, (0x0145) +.equ p1_low_b1, 0x0000 +.equ p1_low_b2, 0x0000 +.equ p1_low_b3, 0x0000 +.equ p1_high_b0, 0x1db7 +.equ p1_high_b1, 0x0000 +.equ p1_high_b2, 0x0000 +.equ p1_high_b3, 0x0000 + +.equ p0_low_b0, (0x0145) +.equ p0_low_b1, 0x0000 +.equ p0_low_b2, 0x0000 +.equ p0_low_b3, 0x0000 +.equ p0_high_b0, 0x0000 +.equ p0_high_b1, 0x0000 +.equ p0_high_b2, 0x0000 +.equ p0_high_b3, 0x0000 + +.equ br_low_b0, (0x001b) +.equ br_low_b1, 0x0000 +.equ br_low_b2, 0x0000 +.equ br_low_b3, 0x0000 +.equ br_high_b0, 0x001b +.equ br_high_b1, 0x0000 +.equ br_high_b2, 0x0000 +.equ br_high_b3, 0x0000 + + .text + .section .rodata + .align 4 + .set .lanchor_crc_tab,. + 0 + .type crc64_tab, %object + .size crc64_tab, 2048 + +crc64_tab: + .xword 0x0000000000000000, 0x000000000000001b + .xword 0x0000000000000036, 0x000000000000002d + .xword 0x000000000000006c, 0x0000000000000077 + .xword 0x000000000000005a, 0x0000000000000041 + .xword 0x00000000000000d8, 0x00000000000000c3 + .xword 0x00000000000000ee, 0x00000000000000f5 + .xword 0x00000000000000b4, 0x00000000000000af + .xword 0x0000000000000082, 0x0000000000000099 + .xword 0x00000000000001b0, 0x00000000000001ab + .xword 0x0000000000000186, 0x000000000000019d + .xword 0x00000000000001dc, 0x00000000000001c7 + .xword 0x00000000000001ea, 0x00000000000001f1 + .xword 0x0000000000000168, 0x0000000000000173 + .xword 0x000000000000015e, 0x0000000000000145 + .xword 0x0000000000000104, 0x000000000000011f + .xword 0x0000000000000132, 0x0000000000000129 + .xword 0x0000000000000360, 0x000000000000037b + .xword 0x0000000000000356, 0x000000000000034d + .xword 0x000000000000030c, 0x0000000000000317 + .xword 0x000000000000033a, 0x0000000000000321 + .xword 0x00000000000003b8, 0x00000000000003a3 + .xword 0x000000000000038e, 0x0000000000000395 + .xword 0x00000000000003d4, 0x00000000000003cf + .xword 0x00000000000003e2, 0x00000000000003f9 + .xword 0x00000000000002d0, 0x00000000000002cb + .xword 0x00000000000002e6, 0x00000000000002fd + .xword 0x00000000000002bc, 0x00000000000002a7 + .xword 0x000000000000028a, 0x0000000000000291 + .xword 0x0000000000000208, 0x0000000000000213 + .xword 0x000000000000023e, 0x0000000000000225 + .xword 0x0000000000000264, 0x000000000000027f + .xword 0x0000000000000252, 0x0000000000000249 + .xword 0x00000000000006c0, 0x00000000000006db + .xword 0x00000000000006f6, 0x00000000000006ed + .xword 0x00000000000006ac, 0x00000000000006b7 + .xword 0x000000000000069a, 0x0000000000000681 + .xword 0x0000000000000618, 0x0000000000000603 + .xword 0x000000000000062e, 0x0000000000000635 + .xword 0x0000000000000674, 0x000000000000066f + .xword 0x0000000000000642, 0x0000000000000659 + .xword 0x0000000000000770, 0x000000000000076b + .xword 0x0000000000000746, 0x000000000000075d + .xword 0x000000000000071c, 0x0000000000000707 + .xword 0x000000000000072a, 0x0000000000000731 + .xword 0x00000000000007a8, 0x00000000000007b3 + .xword 0x000000000000079e, 0x0000000000000785 + .xword 0x00000000000007c4, 0x00000000000007df + .xword 0x00000000000007f2, 0x00000000000007e9 + .xword 0x00000000000005a0, 0x00000000000005bb + .xword 0x0000000000000596, 0x000000000000058d + .xword 0x00000000000005cc, 0x00000000000005d7 + .xword 0x00000000000005fa, 0x00000000000005e1 + .xword 0x0000000000000578, 0x0000000000000563 + .xword 0x000000000000054e, 0x0000000000000555 + .xword 0x0000000000000514, 0x000000000000050f + .xword 0x0000000000000522, 0x0000000000000539 + .xword 0x0000000000000410, 0x000000000000040b + .xword 0x0000000000000426, 0x000000000000043d + .xword 0x000000000000047c, 0x0000000000000467 + .xword 0x000000000000044a, 0x0000000000000451 + .xword 0x00000000000004c8, 0x00000000000004d3 + .xword 0x00000000000004fe, 0x00000000000004e5 + .xword 0x00000000000004a4, 0x00000000000004bf + .xword 0x0000000000000492, 0x0000000000000489 + .xword 0x0000000000000d80, 0x0000000000000d9b + .xword 0x0000000000000db6, 0x0000000000000dad + .xword 0x0000000000000dec, 0x0000000000000df7 + .xword 0x0000000000000dda, 0x0000000000000dc1 + .xword 0x0000000000000d58, 0x0000000000000d43 + .xword 0x0000000000000d6e, 0x0000000000000d75 + .xword 0x0000000000000d34, 0x0000000000000d2f + .xword 0x0000000000000d02, 0x0000000000000d19 + .xword 0x0000000000000c30, 0x0000000000000c2b + .xword 0x0000000000000c06, 0x0000000000000c1d + .xword 0x0000000000000c5c, 0x0000000000000c47 + .xword 0x0000000000000c6a, 0x0000000000000c71 + .xword 0x0000000000000ce8, 0x0000000000000cf3 + .xword 0x0000000000000cde, 0x0000000000000cc5 + .xword 0x0000000000000c84, 0x0000000000000c9f + .xword 0x0000000000000cb2, 0x0000000000000ca9 + .xword 0x0000000000000ee0, 0x0000000000000efb + .xword 0x0000000000000ed6, 0x0000000000000ecd + .xword 0x0000000000000e8c, 0x0000000000000e97 + .xword 0x0000000000000eba, 0x0000000000000ea1 + .xword 0x0000000000000e38, 0x0000000000000e23 + .xword 0x0000000000000e0e, 0x0000000000000e15 + .xword 0x0000000000000e54, 0x0000000000000e4f + .xword 0x0000000000000e62, 0x0000000000000e79 + .xword 0x0000000000000f50, 0x0000000000000f4b + .xword 0x0000000000000f66, 0x0000000000000f7d + .xword 0x0000000000000f3c, 0x0000000000000f27 + .xword 0x0000000000000f0a, 0x0000000000000f11 + .xword 0x0000000000000f88, 0x0000000000000f93 + .xword 0x0000000000000fbe, 0x0000000000000fa5 + .xword 0x0000000000000fe4, 0x0000000000000fff + .xword 0x0000000000000fd2, 0x0000000000000fc9 + .xword 0x0000000000000b40, 0x0000000000000b5b + .xword 0x0000000000000b76, 0x0000000000000b6d + .xword 0x0000000000000b2c, 0x0000000000000b37 + .xword 0x0000000000000b1a, 0x0000000000000b01 + .xword 0x0000000000000b98, 0x0000000000000b83 + .xword 0x0000000000000bae, 0x0000000000000bb5 + .xword 0x0000000000000bf4, 0x0000000000000bef + .xword 0x0000000000000bc2, 0x0000000000000bd9 + .xword 0x0000000000000af0, 0x0000000000000aeb + .xword 0x0000000000000ac6, 0x0000000000000add + .xword 0x0000000000000a9c, 0x0000000000000a87 + .xword 0x0000000000000aaa, 0x0000000000000ab1 + .xword 0x0000000000000a28, 0x0000000000000a33 + .xword 0x0000000000000a1e, 0x0000000000000a05 + .xword 0x0000000000000a44, 0x0000000000000a5f + .xword 0x0000000000000a72, 0x0000000000000a69 + .xword 0x0000000000000820, 0x000000000000083b + .xword 0x0000000000000816, 0x000000000000080d + .xword 0x000000000000084c, 0x0000000000000857 + .xword 0x000000000000087a, 0x0000000000000861 + .xword 0x00000000000008f8, 0x00000000000008e3 + .xword 0x00000000000008ce, 0x00000000000008d5 + .xword 0x0000000000000894, 0x000000000000088f + .xword 0x00000000000008a2, 0x00000000000008b9 + .xword 0x0000000000000990, 0x000000000000098b + .xword 0x00000000000009a6, 0x00000000000009bd + .xword 0x00000000000009fc, 0x00000000000009e7 + .xword 0x00000000000009ca, 0x00000000000009d1 + .xword 0x0000000000000948, 0x0000000000000953 + .xword 0x000000000000097e, 0x0000000000000965 + .xword 0x0000000000000924, 0x000000000000093f + .xword 0x0000000000000912, 0x0000000000000909 diff --git a/src/isa-l/crc/aarch64/crc64_iso_refl_pmull.S b/src/isa-l/crc/aarch64/crc64_iso_refl_pmull.S new file mode 100644 index 000000000..2d2bc6658 --- /dev/null +++ b/src/isa-l/crc/aarch64/crc64_iso_refl_pmull.S @@ -0,0 +1,33 @@ +######################################################################## +# Copyright(c) 2019 Arm Corporation All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Arm Corporation nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +######################################################################### + +#include "crc64_iso_refl_pmull.h" +#include "crc64_refl_common_pmull.h" + +crc64_refl_func crc64_iso_refl_pmull diff --git a/src/isa-l/crc/aarch64/crc64_iso_refl_pmull.h b/src/isa-l/crc/aarch64/crc64_iso_refl_pmull.h new file mode 100644 index 000000000..8ee4f58b1 --- /dev/null +++ b/src/isa-l/crc/aarch64/crc64_iso_refl_pmull.h @@ -0,0 +1,197 @@ +######################################################################## +# Copyright(c) 2019 Arm Corporation All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Arm Corporation nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +######################################################################### + +.equ p4_low_b0, 0x0001 +.equ p4_low_b1, 0xb000 +.equ p4_low_b2, 0x01b1 +.equ p4_low_b3, 0x01b0 +.equ p4_high_b0, 0x0001 +.equ p4_high_b1, 0x0000 +.equ p4_high_b2, 0x0101 +.equ p4_high_b3, 0xb100 + +.equ p1_low_b0, 0x0001 +.equ p1_low_b1, 0x0000 +.equ p1_low_b2, 0x0000 +.equ p1_low_b3, 0x6b70 +.equ p1_high_b0, 0x0001 +.equ p1_high_b1, 0x0000 +.equ p1_high_b2, 0x0000 +.equ p1_high_b3, 0xf500 + +.equ p0_low_b0, 0x0001 +.equ p0_low_b1, 0x0000 +.equ p0_low_b2, 0x0000 +.equ p0_low_b3, 0xf500 + +.equ br_low_b0, 0x0001 +.equ br_low_b1, 0x0000 +.equ br_low_b2, 0x0000 +.equ br_low_b3, 0xb000 +.equ br_high_b0, 0x0001 +.equ br_high_b1, 0x0000 +.equ br_high_b2, 0x0000 +.equ br_high_b3, 0xb000 + + .text + .section .rodata + .align 4 + .set .lanchor_crc_tab,. + 0 + .type crc64_tab, %object + .size crc64_tab, 2048 + +crc64_tab: + .xword 0x0000000000000000, 0x01b0000000000000 + .xword 0x0360000000000000, 0x02d0000000000000 + .xword 0x06c0000000000000, 0x0770000000000000 + .xword 0x05a0000000000000, 0x0410000000000000 + .xword 0x0d80000000000000, 0x0c30000000000000 + .xword 0x0ee0000000000000, 0x0f50000000000000 + .xword 0x0b40000000000000, 0x0af0000000000000 + .xword 0x0820000000000000, 0x0990000000000000 + .xword 0x1b00000000000000, 0x1ab0000000000000 + .xword 0x1860000000000000, 0x19d0000000000000 + .xword 0x1dc0000000000000, 0x1c70000000000000 + .xword 0x1ea0000000000000, 0x1f10000000000000 + .xword 0x1680000000000000, 0x1730000000000000 + .xword 0x15e0000000000000, 0x1450000000000000 + .xword 0x1040000000000000, 0x11f0000000000000 + .xword 0x1320000000000000, 0x1290000000000000 + .xword 0x3600000000000000, 0x37b0000000000000 + .xword 0x3560000000000000, 0x34d0000000000000 + .xword 0x30c0000000000000, 0x3170000000000000 + .xword 0x33a0000000000000, 0x3210000000000000 + .xword 0x3b80000000000000, 0x3a30000000000000 + .xword 0x38e0000000000000, 0x3950000000000000 + .xword 0x3d40000000000000, 0x3cf0000000000000 + .xword 0x3e20000000000000, 0x3f90000000000000 + .xword 0x2d00000000000000, 0x2cb0000000000000 + .xword 0x2e60000000000000, 0x2fd0000000000000 + .xword 0x2bc0000000000000, 0x2a70000000000000 + .xword 0x28a0000000000000, 0x2910000000000000 + .xword 0x2080000000000000, 0x2130000000000000 + .xword 0x23e0000000000000, 0x2250000000000000 + .xword 0x2640000000000000, 0x27f0000000000000 + .xword 0x2520000000000000, 0x2490000000000000 + .xword 0x6c00000000000000, 0x6db0000000000000 + .xword 0x6f60000000000000, 0x6ed0000000000000 + .xword 0x6ac0000000000000, 0x6b70000000000000 + .xword 0x69a0000000000000, 0x6810000000000000 + .xword 0x6180000000000000, 0x6030000000000000 + .xword 0x62e0000000000000, 0x6350000000000000 + .xword 0x6740000000000000, 0x66f0000000000000 + .xword 0x6420000000000000, 0x6590000000000000 + .xword 0x7700000000000000, 0x76b0000000000000 + .xword 0x7460000000000000, 0x75d0000000000000 + .xword 0x71c0000000000000, 0x7070000000000000 + .xword 0x72a0000000000000, 0x7310000000000000 + .xword 0x7a80000000000000, 0x7b30000000000000 + .xword 0x79e0000000000000, 0x7850000000000000 + .xword 0x7c40000000000000, 0x7df0000000000000 + .xword 0x7f20000000000000, 0x7e90000000000000 + .xword 0x5a00000000000000, 0x5bb0000000000000 + .xword 0x5960000000000000, 0x58d0000000000000 + .xword 0x5cc0000000000000, 0x5d70000000000000 + .xword 0x5fa0000000000000, 0x5e10000000000000 + .xword 0x5780000000000000, 0x5630000000000000 + .xword 0x54e0000000000000, 0x5550000000000000 + .xword 0x5140000000000000, 0x50f0000000000000 + .xword 0x5220000000000000, 0x5390000000000000 + .xword 0x4100000000000000, 0x40b0000000000000 + .xword 0x4260000000000000, 0x43d0000000000000 + .xword 0x47c0000000000000, 0x4670000000000000 + .xword 0x44a0000000000000, 0x4510000000000000 + .xword 0x4c80000000000000, 0x4d30000000000000 + .xword 0x4fe0000000000000, 0x4e50000000000000 + .xword 0x4a40000000000000, 0x4bf0000000000000 + .xword 0x4920000000000000, 0x4890000000000000 + .xword 0xd800000000000000, 0xd9b0000000000000 + .xword 0xdb60000000000000, 0xdad0000000000000 + .xword 0xdec0000000000000, 0xdf70000000000000 + .xword 0xdda0000000000000, 0xdc10000000000000 + .xword 0xd580000000000000, 0xd430000000000000 + .xword 0xd6e0000000000000, 0xd750000000000000 + .xword 0xd340000000000000, 0xd2f0000000000000 + .xword 0xd020000000000000, 0xd190000000000000 + .xword 0xc300000000000000, 0xc2b0000000000000 + .xword 0xc060000000000000, 0xc1d0000000000000 + .xword 0xc5c0000000000000, 0xc470000000000000 + .xword 0xc6a0000000000000, 0xc710000000000000 + .xword 0xce80000000000000, 0xcf30000000000000 + .xword 0xcde0000000000000, 0xcc50000000000000 + .xword 0xc840000000000000, 0xc9f0000000000000 + .xword 0xcb20000000000000, 0xca90000000000000 + .xword 0xee00000000000000, 0xefb0000000000000 + .xword 0xed60000000000000, 0xecd0000000000000 + .xword 0xe8c0000000000000, 0xe970000000000000 + .xword 0xeba0000000000000, 0xea10000000000000 + .xword 0xe380000000000000, 0xe230000000000000 + .xword 0xe0e0000000000000, 0xe150000000000000 + .xword 0xe540000000000000, 0xe4f0000000000000 + .xword 0xe620000000000000, 0xe790000000000000 + .xword 0xf500000000000000, 0xf4b0000000000000 + .xword 0xf660000000000000, 0xf7d0000000000000 + .xword 0xf3c0000000000000, 0xf270000000000000 + .xword 0xf0a0000000000000, 0xf110000000000000 + .xword 0xf880000000000000, 0xf930000000000000 + .xword 0xfbe0000000000000, 0xfa50000000000000 + .xword 0xfe40000000000000, 0xfff0000000000000 + .xword 0xfd20000000000000, 0xfc90000000000000 + .xword 0xb400000000000000, 0xb5b0000000000000 + .xword 0xb760000000000000, 0xb6d0000000000000 + .xword 0xb2c0000000000000, 0xb370000000000000 + .xword 0xb1a0000000000000, 0xb010000000000000 + .xword 0xb980000000000000, 0xb830000000000000 + .xword 0xbae0000000000000, 0xbb50000000000000 + .xword 0xbf40000000000000, 0xbef0000000000000 + .xword 0xbc20000000000000, 0xbd90000000000000 + .xword 0xaf00000000000000, 0xaeb0000000000000 + .xword 0xac60000000000000, 0xadd0000000000000 + .xword 0xa9c0000000000000, 0xa870000000000000 + .xword 0xaaa0000000000000, 0xab10000000000000 + .xword 0xa280000000000000, 0xa330000000000000 + .xword 0xa1e0000000000000, 0xa050000000000000 + .xword 0xa440000000000000, 0xa5f0000000000000 + .xword 0xa720000000000000, 0xa690000000000000 + .xword 0x8200000000000000, 0x83b0000000000000 + .xword 0x8160000000000000, 0x80d0000000000000 + .xword 0x84c0000000000000, 0x8570000000000000 + .xword 0x87a0000000000000, 0x8610000000000000 + .xword 0x8f80000000000000, 0x8e30000000000000 + .xword 0x8ce0000000000000, 0x8d50000000000000 + .xword 0x8940000000000000, 0x88f0000000000000 + .xword 0x8a20000000000000, 0x8b90000000000000 + .xword 0x9900000000000000, 0x98b0000000000000 + .xword 0x9a60000000000000, 0x9bd0000000000000 + .xword 0x9fc0000000000000, 0x9e70000000000000 + .xword 0x9ca0000000000000, 0x9d10000000000000 + .xword 0x9480000000000000, 0x9530000000000000 + .xword 0x97e0000000000000, 0x9650000000000000 + .xword 0x9240000000000000, 0x93f0000000000000 + .xword 0x9120000000000000, 0x9090000000000000 diff --git a/src/isa-l/crc/aarch64/crc64_jones_norm_pmull.S b/src/isa-l/crc/aarch64/crc64_jones_norm_pmull.S new file mode 100644 index 000000000..4f298376c --- /dev/null +++ b/src/isa-l/crc/aarch64/crc64_jones_norm_pmull.S @@ -0,0 +1,33 @@ +######################################################################## +# Copyright(c) 2019 Arm Corporation All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Arm Corporation nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +######################################################################### + +#include "crc64_jones_norm_pmull.h" +#include "crc64_norm_common_pmull.h" + +crc64_norm_func crc64_jones_norm_pmull diff --git a/src/isa-l/crc/aarch64/crc64_jones_norm_pmull.h b/src/isa-l/crc/aarch64/crc64_jones_norm_pmull.h new file mode 100644 index 000000000..a20fa045d --- /dev/null +++ b/src/isa-l/crc/aarch64/crc64_jones_norm_pmull.h @@ -0,0 +1,200 @@ +######################################################################## +# Copyright(c) 2019 Arm Corporation All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Arm Corporation nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +######################################################################### + +.equ p4_low_b0, (0xd25e) +.equ p4_low_b1, 0xca43 +.equ p4_low_b2, 0x1e58 +.equ p4_low_b3, 0x4e50 +.equ p4_high_b0, 0xf643 +.equ p4_high_b1, 0x8f27 +.equ p4_high_b2, 0x6158 +.equ p4_high_b3, 0x13c9 + +.equ p1_low_b0, (0x7038) +.equ p1_low_b1, 0x5001 +.equ p1_low_b2, 0xed27 +.equ p1_low_b3, 0x4445 +.equ p1_high_b0, 0xd736 +.equ p1_high_b1, 0x7cfb +.equ p1_high_b2, 0x7415 +.equ p1_high_b3, 0x698b + +.equ p0_low_b0, (0x7038) +.equ p0_low_b1, 0x5001 +.equ p0_low_b2, 0xed27 +.equ p0_low_b3, 0x4445 +.equ p0_high_b0, 0x0000 +.equ p0_high_b1, 0x0000 +.equ p0_high_b2, 0x0000 +.equ p0_high_b3, 0x0000 + +.equ br_low_b0, (0x6cf8) +.equ br_low_b1, 0x98be +.equ br_low_b2, 0xeeb2 +.equ br_low_b3, 0xddf3 +.equ br_high_b0, 0x35a9 +.equ br_high_b1, 0x94c9 +.equ br_high_b2, 0xd235 +.equ br_high_b3, 0xad93 + + .text + .section .rodata + .align 4 + .set .lanchor_crc_tab,. + 0 + .type crc64_tab, %object + .size crc64_tab, 2048 +crc64_tab: + .xword 0x0000000000000000, 0xad93d23594c935a9 + .xword 0xf6b4765ebd5b5efb, 0x5b27a46b29926b52 + .xword 0x40fb3e88ee7f885f, 0xed68ecbd7ab6bdf6 + .xword 0xb64f48d65324d6a4, 0x1bdc9ae3c7ede30d + .xword 0x81f67d11dcff10be, 0x2c65af2448362517 + .xword 0x77420b4f61a44e45, 0xdad1d97af56d7bec + .xword 0xc10d4399328098e1, 0x6c9e91aca649ad48 + .xword 0x37b935c78fdbc61a, 0x9a2ae7f21b12f3b3 + .xword 0xae7f28162d3714d5, 0x03ecfa23b9fe217c + .xword 0x58cb5e48906c4a2e, 0xf5588c7d04a57f87 + .xword 0xee84169ec3489c8a, 0x4317c4ab5781a923 + .xword 0x183060c07e13c271, 0xb5a3b2f5eadaf7d8 + .xword 0x2f895507f1c8046b, 0x821a8732650131c2 + .xword 0xd93d23594c935a90, 0x74aef16cd85a6f39 + .xword 0x6f726b8f1fb78c34, 0xc2e1b9ba8b7eb99d + .xword 0x99c61dd1a2ecd2cf, 0x3455cfe43625e766 + .xword 0xf16d8219cea71c03, 0x5cfe502c5a6e29aa + .xword 0x07d9f44773fc42f8, 0xaa4a2672e7357751 + .xword 0xb196bc9120d8945c, 0x1c056ea4b411a1f5 + .xword 0x4722cacf9d83caa7, 0xeab118fa094aff0e + .xword 0x709bff0812580cbd, 0xdd082d3d86913914 + .xword 0x862f8956af035246, 0x2bbc5b633bca67ef + .xword 0x3060c180fc2784e2, 0x9df313b568eeb14b + .xword 0xc6d4b7de417cda19, 0x6b4765ebd5b5efb0 + .xword 0x5f12aa0fe39008d6, 0xf281783a77593d7f + .xword 0xa9a6dc515ecb562d, 0x04350e64ca026384 + .xword 0x1fe994870def8089, 0xb27a46b29926b520 + .xword 0xe95de2d9b0b4de72, 0x44ce30ec247debdb + .xword 0xdee4d71e3f6f1868, 0x7377052baba62dc1 + .xword 0x2850a14082344693, 0x85c3737516fd733a + .xword 0x9e1fe996d1109037, 0x338c3ba345d9a59e + .xword 0x68ab9fc86c4bcecc, 0xc5384dfdf882fb65 + .xword 0x4f48d60609870daf, 0xe2db04339d4e3806 + .xword 0xb9fca058b4dc5354, 0x146f726d201566fd + .xword 0x0fb3e88ee7f885f0, 0xa2203abb7331b059 + .xword 0xf9079ed05aa3db0b, 0x54944ce5ce6aeea2 + .xword 0xcebeab17d5781d11, 0x632d792241b128b8 + .xword 0x380add49682343ea, 0x95990f7cfcea7643 + .xword 0x8e45959f3b07954e, 0x23d647aaafcea0e7 + .xword 0x78f1e3c1865ccbb5, 0xd56231f41295fe1c + .xword 0xe137fe1024b0197a, 0x4ca42c25b0792cd3 + .xword 0x1783884e99eb4781, 0xba105a7b0d227228 + .xword 0xa1ccc098cacf9125, 0x0c5f12ad5e06a48c + .xword 0x5778b6c67794cfde, 0xfaeb64f3e35dfa77 + .xword 0x60c18301f84f09c4, 0xcd5251346c863c6d + .xword 0x9675f55f4514573f, 0x3be6276ad1dd6296 + .xword 0x203abd891630819b, 0x8da96fbc82f9b432 + .xword 0xd68ecbd7ab6bdf60, 0x7b1d19e23fa2eac9 + .xword 0xbe25541fc72011ac, 0x13b6862a53e92405 + .xword 0x489122417a7b4f57, 0xe502f074eeb27afe + .xword 0xfede6a97295f99f3, 0x534db8a2bd96ac5a + .xword 0x086a1cc99404c708, 0xa5f9cefc00cdf2a1 + .xword 0x3fd3290e1bdf0112, 0x9240fb3b8f1634bb + .xword 0xc9675f50a6845fe9, 0x64f48d65324d6a40 + .xword 0x7f281786f5a0894d, 0xd2bbc5b36169bce4 + .xword 0x899c61d848fbd7b6, 0x240fb3eddc32e21f + .xword 0x105a7c09ea170579, 0xbdc9ae3c7ede30d0 + .xword 0xe6ee0a57574c5b82, 0x4b7dd862c3856e2b + .xword 0x50a1428104688d26, 0xfd3290b490a1b88f + .xword 0xa61534dfb933d3dd, 0x0b86e6ea2dfae674 + .xword 0x91ac011836e815c7, 0x3c3fd32da221206e + .xword 0x671877468bb34b3c, 0xca8ba5731f7a7e95 + .xword 0xd1573f90d8979d98, 0x7cc4eda54c5ea831 + .xword 0x27e349ce65ccc363, 0x8a709bfbf105f6ca + .xword 0x9e91ac0c130e1b5e, 0x33027e3987c72ef7 + .xword 0x6825da52ae5545a5, 0xc5b608673a9c700c + .xword 0xde6a9284fd719301, 0x73f940b169b8a6a8 + .xword 0x28dee4da402acdfa, 0x854d36efd4e3f853 + .xword 0x1f67d11dcff10be0, 0xb2f403285b383e49 + .xword 0xe9d3a74372aa551b, 0x44407576e66360b2 + .xword 0x5f9cef95218e83bf, 0xf20f3da0b547b616 + .xword 0xa92899cb9cd5dd44, 0x04bb4bfe081ce8ed + .xword 0x30ee841a3e390f8b, 0x9d7d562faaf03a22 + .xword 0xc65af24483625170, 0x6bc9207117ab64d9 + .xword 0x7015ba92d04687d4, 0xdd8668a7448fb27d + .xword 0x86a1cccc6d1dd92f, 0x2b321ef9f9d4ec86 + .xword 0xb118f90be2c61f35, 0x1c8b2b3e760f2a9c + .xword 0x47ac8f555f9d41ce, 0xea3f5d60cb547467 + .xword 0xf1e3c7830cb9976a, 0x5c7015b69870a2c3 + .xword 0x0757b1ddb1e2c991, 0xaac463e8252bfc38 + .xword 0x6ffc2e15dda9075d, 0xc26ffc20496032f4 + .xword 0x9948584b60f259a6, 0x34db8a7ef43b6c0f + .xword 0x2f07109d33d68f02, 0x8294c2a8a71fbaab + .xword 0xd9b366c38e8dd1f9, 0x7420b4f61a44e450 + .xword 0xee0a5304015617e3, 0x43998131959f224a + .xword 0x18be255abc0d4918, 0xb52df76f28c47cb1 + .xword 0xaef16d8cef299fbc, 0x0362bfb97be0aa15 + .xword 0x58451bd25272c147, 0xf5d6c9e7c6bbf4ee + .xword 0xc1830603f09e1388, 0x6c10d43664572621 + .xword 0x3737705d4dc54d73, 0x9aa4a268d90c78da + .xword 0x8178388b1ee19bd7, 0x2cebeabe8a28ae7e + .xword 0x77cc4ed5a3bac52c, 0xda5f9ce03773f085 + .xword 0x40757b122c610336, 0xede6a927b8a8369f + .xword 0xb6c10d4c913a5dcd, 0x1b52df7905f36864 + .xword 0x008e459ac21e8b69, 0xad1d97af56d7bec0 + .xword 0xf63a33c47f45d592, 0x5ba9e1f1eb8ce03b + .xword 0xd1d97a0a1a8916f1, 0x7c4aa83f8e402358 + .xword 0x276d0c54a7d2480a, 0x8afede61331b7da3 + .xword 0x91224482f4f69eae, 0x3cb196b7603fab07 + .xword 0x679632dc49adc055, 0xca05e0e9dd64f5fc + .xword 0x502f071bc676064f, 0xfdbcd52e52bf33e6 + .xword 0xa69b71457b2d58b4, 0x0b08a370efe46d1d + .xword 0x10d4399328098e10, 0xbd47eba6bcc0bbb9 + .xword 0xe6604fcd9552d0eb, 0x4bf39df8019be542 + .xword 0x7fa6521c37be0224, 0xd2358029a377378d + .xword 0x891224428ae55cdf, 0x2481f6771e2c6976 + .xword 0x3f5d6c94d9c18a7b, 0x92cebea14d08bfd2 + .xword 0xc9e91aca649ad480, 0x647ac8fff053e129 + .xword 0xfe502f0deb41129a, 0x53c3fd387f882733 + .xword 0x08e45953561a4c61, 0xa5778b66c2d379c8 + .xword 0xbeab1185053e9ac5, 0x1338c3b091f7af6c + .xword 0x481f67dbb865c43e, 0xe58cb5ee2cacf197 + .xword 0x20b4f813d42e0af2, 0x8d272a2640e73f5b + .xword 0xd6008e4d69755409, 0x7b935c78fdbc61a0 + .xword 0x604fc69b3a5182ad, 0xcddc14aeae98b704 + .xword 0x96fbb0c5870adc56, 0x3b6862f013c3e9ff + .xword 0xa142850208d11a4c, 0x0cd157379c182fe5 + .xword 0x57f6f35cb58a44b7, 0xfa6521692143711e + .xword 0xe1b9bb8ae6ae9213, 0x4c2a69bf7267a7ba + .xword 0x170dcdd45bf5cce8, 0xba9e1fe1cf3cf941 + .xword 0x8ecbd005f9191e27, 0x235802306dd02b8e + .xword 0x787fa65b444240dc, 0xd5ec746ed08b7575 + .xword 0xce30ee8d17669678, 0x63a33cb883afa3d1 + .xword 0x388498d3aa3dc883, 0x95174ae63ef4fd2a + .xword 0x0f3dad1425e60e99, 0xa2ae7f21b12f3b30 + .xword 0xf989db4a98bd5062, 0x541a097f0c7465cb + .xword 0x4fc6939ccb9986c6, 0xe25541a95f50b36f + .xword 0xb972e5c276c2d83d, 0x14e137f7e20bed94 diff --git a/src/isa-l/crc/aarch64/crc64_jones_refl_pmull.S b/src/isa-l/crc/aarch64/crc64_jones_refl_pmull.S new file mode 100644 index 000000000..177092f9f --- /dev/null +++ b/src/isa-l/crc/aarch64/crc64_jones_refl_pmull.S @@ -0,0 +1,33 @@ +######################################################################## +# Copyright(c) 2019 Arm Corporation All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Arm Corporation nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +######################################################################### + +#include "crc64_jones_refl_pmull.h" +#include "crc64_refl_common_pmull.h" + +crc64_refl_func crc64_jones_refl_pmull diff --git a/src/isa-l/crc/aarch64/crc64_jones_refl_pmull.h b/src/isa-l/crc/aarch64/crc64_jones_refl_pmull.h new file mode 100644 index 000000000..5bf98f73e --- /dev/null +++ b/src/isa-l/crc/aarch64/crc64_jones_refl_pmull.h @@ -0,0 +1,196 @@ +######################################################################## +# Copyright(c) 2019 Arm Corporation All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Arm Corporation nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +######################################################################### + +.equ p4_low_b0, 0xb4fb +.equ p4_low_b1, 0x6d9a +.equ p4_low_b2, 0xefb1 +.equ p4_low_b3, 0xaf86 +.equ p4_high_b0, 0x14e4 +.equ p4_high_b1, 0x34f0 +.equ p4_high_b2, 0x84a6 +.equ p4_high_b3, 0xf497 + +.equ p1_low_b0, 0xa32c +.equ p1_low_b1, 0x505d +.equ p1_low_b2, 0xbe7d +.equ p1_low_b3, 0xd9d7 +.equ p1_high_b0, 0x4444 +.equ p1_high_b1, 0xc96f +.equ p1_high_b2, 0x0015 +.equ p1_high_b3, 0x381d + +.equ p0_low_b0, 0x4444 +.equ p0_low_b1, 0xc96f +.equ p0_low_b2, 0x0015 +.equ p0_low_b3, 0x381d + +.equ br_low_b0, 0x9f77 +.equ br_low_b1, 0x9aef +.equ br_low_b2, 0xfa32 +.equ br_low_b3, 0x3e6c +.equ br_high_b0, 0x936b +.equ br_high_b1, 0x5897 +.equ br_high_b2, 0x2653 +.equ br_high_b3, 0x2b59 + + .text + .section .rodata + .align 4 + .set .lanchor_crc_tab,. + 0 + .type crc64_tab, %object + .size crc64_tab, 2048 +crc64_tab: + .xword 0x0000000000000000, 0x7ad870c830358979 + .xword 0xf5b0e190606b12f2, 0x8f689158505e9b8b + .xword 0xc038e5739841b68f, 0xbae095bba8743ff6 + .xword 0x358804e3f82aa47d, 0x4f50742bc81f2d04 + .xword 0xab28ecb46814fe75, 0xd1f09c7c5821770c + .xword 0x5e980d24087fec87, 0x24407dec384a65fe + .xword 0x6b1009c7f05548fa, 0x11c8790fc060c183 + .xword 0x9ea0e857903e5a08, 0xe478989fa00bd371 + .xword 0x7d08ff3b88be6f81, 0x07d08ff3b88be6f8 + .xword 0x88b81eabe8d57d73, 0xf2606e63d8e0f40a + .xword 0xbd301a4810ffd90e, 0xc7e86a8020ca5077 + .xword 0x4880fbd87094cbfc, 0x32588b1040a14285 + .xword 0xd620138fe0aa91f4, 0xacf86347d09f188d + .xword 0x2390f21f80c18306, 0x594882d7b0f40a7f + .xword 0x1618f6fc78eb277b, 0x6cc0863448deae02 + .xword 0xe3a8176c18803589, 0x997067a428b5bcf0 + .xword 0xfa11fe77117cdf02, 0x80c98ebf2149567b + .xword 0x0fa11fe77117cdf0, 0x75796f2f41224489 + .xword 0x3a291b04893d698d, 0x40f16bccb908e0f4 + .xword 0xcf99fa94e9567b7f, 0xb5418a5cd963f206 + .xword 0x513912c379682177, 0x2be1620b495da80e + .xword 0xa489f35319033385, 0xde51839b2936bafc + .xword 0x9101f7b0e12997f8, 0xebd98778d11c1e81 + .xword 0x64b116208142850a, 0x1e6966e8b1770c73 + .xword 0x8719014c99c2b083, 0xfdc17184a9f739fa + .xword 0x72a9e0dcf9a9a271, 0x08719014c99c2b08 + .xword 0x4721e43f0183060c, 0x3df994f731b68f75 + .xword 0xb29105af61e814fe, 0xc849756751dd9d87 + .xword 0x2c31edf8f1d64ef6, 0x56e99d30c1e3c78f + .xword 0xd9810c6891bd5c04, 0xa3597ca0a188d57d + .xword 0xec09088b6997f879, 0x96d1784359a27100 + .xword 0x19b9e91b09fcea8b, 0x636199d339c963f2 + .xword 0xdf7adabd7a6e2d6f, 0xa5a2aa754a5ba416 + .xword 0x2aca3b2d1a053f9d, 0x50124be52a30b6e4 + .xword 0x1f423fcee22f9be0, 0x659a4f06d21a1299 + .xword 0xeaf2de5e82448912, 0x902aae96b271006b + .xword 0x74523609127ad31a, 0x0e8a46c1224f5a63 + .xword 0x81e2d7997211c1e8, 0xfb3aa75142244891 + .xword 0xb46ad37a8a3b6595, 0xceb2a3b2ba0eecec + .xword 0x41da32eaea507767, 0x3b024222da65fe1e + .xword 0xa2722586f2d042ee, 0xd8aa554ec2e5cb97 + .xword 0x57c2c41692bb501c, 0x2d1ab4dea28ed965 + .xword 0x624ac0f56a91f461, 0x1892b03d5aa47d18 + .xword 0x97fa21650afae693, 0xed2251ad3acf6fea + .xword 0x095ac9329ac4bc9b, 0x7382b9faaaf135e2 + .xword 0xfcea28a2faafae69, 0x8632586aca9a2710 + .xword 0xc9622c4102850a14, 0xb3ba5c8932b0836d + .xword 0x3cd2cdd162ee18e6, 0x460abd1952db919f + .xword 0x256b24ca6b12f26d, 0x5fb354025b277b14 + .xword 0xd0dbc55a0b79e09f, 0xaa03b5923b4c69e6 + .xword 0xe553c1b9f35344e2, 0x9f8bb171c366cd9b + .xword 0x10e3202993385610, 0x6a3b50e1a30ddf69 + .xword 0x8e43c87e03060c18, 0xf49bb8b633338561 + .xword 0x7bf329ee636d1eea, 0x012b592653589793 + .xword 0x4e7b2d0d9b47ba97, 0x34a35dc5ab7233ee + .xword 0xbbcbcc9dfb2ca865, 0xc113bc55cb19211c + .xword 0x5863dbf1e3ac9dec, 0x22bbab39d3991495 + .xword 0xadd33a6183c78f1e, 0xd70b4aa9b3f20667 + .xword 0x985b3e827bed2b63, 0xe2834e4a4bd8a21a + .xword 0x6debdf121b863991, 0x1733afda2bb3b0e8 + .xword 0xf34b37458bb86399, 0x8993478dbb8deae0 + .xword 0x06fbd6d5ebd3716b, 0x7c23a61ddbe6f812 + .xword 0x3373d23613f9d516, 0x49aba2fe23cc5c6f + .xword 0xc6c333a67392c7e4, 0xbc1b436e43a74e9d + .xword 0x95ac9329ac4bc9b5, 0xef74e3e19c7e40cc + .xword 0x601c72b9cc20db47, 0x1ac40271fc15523e + .xword 0x5594765a340a7f3a, 0x2f4c0692043ff643 + .xword 0xa02497ca54616dc8, 0xdafce7026454e4b1 + .xword 0x3e847f9dc45f37c0, 0x445c0f55f46abeb9 + .xword 0xcb349e0da4342532, 0xb1eceec59401ac4b + .xword 0xfebc9aee5c1e814f, 0x8464ea266c2b0836 + .xword 0x0b0c7b7e3c7593bd, 0x71d40bb60c401ac4 + .xword 0xe8a46c1224f5a634, 0x927c1cda14c02f4d + .xword 0x1d148d82449eb4c6, 0x67ccfd4a74ab3dbf + .xword 0x289c8961bcb410bb, 0x5244f9a98c8199c2 + .xword 0xdd2c68f1dcdf0249, 0xa7f41839ecea8b30 + .xword 0x438c80a64ce15841, 0x3954f06e7cd4d138 + .xword 0xb63c61362c8a4ab3, 0xcce411fe1cbfc3ca + .xword 0x83b465d5d4a0eece, 0xf96c151de49567b7 + .xword 0x76048445b4cbfc3c, 0x0cdcf48d84fe7545 + .xword 0x6fbd6d5ebd3716b7, 0x15651d968d029fce + .xword 0x9a0d8ccedd5c0445, 0xe0d5fc06ed698d3c + .xword 0xaf85882d2576a038, 0xd55df8e515432941 + .xword 0x5a3569bd451db2ca, 0x20ed197575283bb3 + .xword 0xc49581ead523e8c2, 0xbe4df122e51661bb + .xword 0x3125607ab548fa30, 0x4bfd10b2857d7349 + .xword 0x04ad64994d625e4d, 0x7e7514517d57d734 + .xword 0xf11d85092d094cbf, 0x8bc5f5c11d3cc5c6 + .xword 0x12b5926535897936, 0x686de2ad05bcf04f + .xword 0xe70573f555e26bc4, 0x9ddd033d65d7e2bd + .xword 0xd28d7716adc8cfb9, 0xa85507de9dfd46c0 + .xword 0x273d9686cda3dd4b, 0x5de5e64efd965432 + .xword 0xb99d7ed15d9d8743, 0xc3450e196da80e3a + .xword 0x4c2d9f413df695b1, 0x36f5ef890dc31cc8 + .xword 0x79a59ba2c5dc31cc, 0x037deb6af5e9b8b5 + .xword 0x8c157a32a5b7233e, 0xf6cd0afa9582aa47 + .xword 0x4ad64994d625e4da, 0x300e395ce6106da3 + .xword 0xbf66a804b64ef628, 0xc5bed8cc867b7f51 + .xword 0x8aeeace74e645255, 0xf036dc2f7e51db2c + .xword 0x7f5e4d772e0f40a7, 0x05863dbf1e3ac9de + .xword 0xe1fea520be311aaf, 0x9b26d5e88e0493d6 + .xword 0x144e44b0de5a085d, 0x6e963478ee6f8124 + .xword 0x21c640532670ac20, 0x5b1e309b16452559 + .xword 0xd476a1c3461bbed2, 0xaeaed10b762e37ab + .xword 0x37deb6af5e9b8b5b, 0x4d06c6676eae0222 + .xword 0xc26e573f3ef099a9, 0xb8b627f70ec510d0 + .xword 0xf7e653dcc6da3dd4, 0x8d3e2314f6efb4ad + .xword 0x0256b24ca6b12f26, 0x788ec2849684a65f + .xword 0x9cf65a1b368f752e, 0xe62e2ad306bafc57 + .xword 0x6946bb8b56e467dc, 0x139ecb4366d1eea5 + .xword 0x5ccebf68aecec3a1, 0x2616cfa09efb4ad8 + .xword 0xa97e5ef8cea5d153, 0xd3a62e30fe90582a + .xword 0xb0c7b7e3c7593bd8, 0xca1fc72bf76cb2a1 + .xword 0x45775673a732292a, 0x3faf26bb9707a053 + .xword 0x70ff52905f188d57, 0x0a2722586f2d042e + .xword 0x854fb3003f739fa5, 0xff97c3c80f4616dc + .xword 0x1bef5b57af4dc5ad, 0x61372b9f9f784cd4 + .xword 0xee5fbac7cf26d75f, 0x9487ca0fff135e26 + .xword 0xdbd7be24370c7322, 0xa10fceec0739fa5b + .xword 0x2e675fb4576761d0, 0x54bf2f7c6752e8a9 + .xword 0xcdcf48d84fe75459, 0xb71738107fd2dd20 + .xword 0x387fa9482f8c46ab, 0x42a7d9801fb9cfd2 + .xword 0x0df7adabd7a6e2d6, 0x772fdd63e7936baf + .xword 0xf8474c3bb7cdf024, 0x829f3cf387f8795d + .xword 0x66e7a46c27f3aa2c, 0x1c3fd4a417c62355 + .xword 0x935745fc4798b8de, 0xe98f353477ad31a7 + .xword 0xa6df411fbfb21ca3, 0xdc0731d78f8795da + .xword 0x536fa08fdfd90e51, 0x29b7d047efec8728 diff --git a/src/isa-l/crc/aarch64/crc64_norm_common_pmull.h b/src/isa-l/crc/aarch64/crc64_norm_common_pmull.h new file mode 100644 index 000000000..1bdfc26b5 --- /dev/null +++ b/src/isa-l/crc/aarch64/crc64_norm_common_pmull.h @@ -0,0 +1,129 @@ +######################################################################## +# Copyright(c) 2019 Arm Corporation All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Arm Corporation nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +######################################################################### + +#include "crc_common_pmull.h" + +.macro crc64_norm_func name:req + .arch armv8-a+crypto + .text + .align 3 + .global \name + .type \name, %function + +/* uint64_t crc64_norm_func(uint64_t seed, const uint8_t * buf, uint64_t len) */ + +\name\(): + mvn x_seed, x_seed + mov x_counter, 0 + cmp x_len, (FOLD_SIZE-1) + bhi .crc_clmul_pre + +.crc_tab_pre: + cmp x_len, x_counter + bls .done + + adrp x_tmp, .lanchor_crc_tab + add x_buf_iter, x_buf, x_counter + add x_buf, x_buf, x_len + add x_crc_tab_addr, x_tmp, :lo12:.lanchor_crc_tab + + .align 3 +.loop_crc_tab: + ldrb w_tmp, [x_buf_iter], 1 + cmp x_buf, x_buf_iter + eor x_tmp, x_tmp, x_seed, lsr 56 + ldr x_tmp, [x_crc_tab_addr, x_tmp, lsl 3] + eor x_seed, x_tmp, x_seed, lsl 8 + bne .loop_crc_tab + +.done: + mvn x_crc_ret, x_seed + ret + + .align 2 +.crc_clmul_pre: + movi v_x0.2s, 0 + fmov v_x0.d[1], x_seed // save crc to v_x0 + + crc_norm_load_first_block + + bls .clmul_loop_end + + crc64_load_p4 + +// 1024bit --> 512bit loop +// merge x0, x1, x2, x3, y0, y1, y2, y3 => x0, x1, x2, x3 (uint64x2_t) + crc_norm_loop + +.clmul_loop_end: +// folding 512bit --> 128bit + crc64_fold_512b_to_128b + +// folding 128bit --> 64bit + mov x_tmp, p0_low_b0 + movk x_tmp, p0_low_b1, lsl 16 + movk x_tmp, p0_low_b2, lsl 32 + movk x_tmp, p0_low_b3, lsl 48 + fmov d_p0_high, x_tmp + + pmull2 v_tmp_high.1q, v_x3.2d, v_p0.2d + movi v_tmp_low.2s, 0 + ext v_tmp_low.16b, v_tmp_low.16b, v_x3.16b, #8 + + eor v_x3.16b, v_tmp_high.16b, v_tmp_low.16b + +// barrett reduction + mov x_tmp, br_low_b0 + movk x_tmp, br_low_b1, lsl 16 + movk x_tmp, br_low_b2, lsl 32 + movk x_tmp, br_low_b3, lsl 48 + fmov d_br_low2, x_tmp + + mov x_tmp2, br_high_b0 + movk x_tmp2, br_high_b1, lsl 16 + movk x_tmp2, br_high_b2, lsl 32 + movk x_tmp2, br_high_b3, lsl 48 + fmov d_br_high2, x_tmp2 + + pmull2 v_tmp_low.1q, v_x3.2d, v_br_low.2d + eor v_tmp_low.16b, v_x3.16b, v_tmp_low.16b + pmull2 v_tmp_low.1q, v_tmp_low.2d, v_br_high.2d + eor v_x3.8b, v_x3.8b, v_tmp_low.8b + umov x_seed, v_x3.d[0] + + b .crc_tab_pre + + .size \name, .-\name + + .section .rodata.cst16,"aM",@progbits,16 + .align 4 +.shuffle_data: + .byte 15, 14, 13, 12, 11, 10, 9, 8 + .byte 7, 6, 5, 4, 3, 2, 1, 0 +.endm diff --git a/src/isa-l/crc/aarch64/crc64_refl_common_pmull.h b/src/isa-l/crc/aarch64/crc64_refl_common_pmull.h new file mode 100644 index 000000000..a45501300 --- /dev/null +++ b/src/isa-l/crc/aarch64/crc64_refl_common_pmull.h @@ -0,0 +1,126 @@ +######################################################################## +# Copyright(c) 2019 Arm Corporation All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Arm Corporation nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +######################################################################### + +#include "crc_common_pmull.h" + +.macro crc64_refl_func name:req + .arch armv8-a+crypto + .text + .align 3 + .global \name + .type \name, %function + +/* uint64_t crc64_refl_func(uint64_t seed, const uint8_t * buf, uint64_t len) */ + +\name\(): + mvn x_seed, x_seed + mov x_counter, 0 + cmp x_len, (FOLD_SIZE-1) + bhi .crc_clmul_pre + +.crc_tab_pre: + cmp x_len, x_counter + bls .done + + adrp x_tmp, .lanchor_crc_tab + add x_buf_iter, x_buf, x_counter + add x_buf, x_buf, x_len + add x_crc_tab_addr, x_tmp, :lo12:.lanchor_crc_tab + + .align 3 +.loop_crc_tab: + ldrb w_tmp, [x_buf_iter], 1 + eor w_tmp, w_tmp, w0 + cmp x_buf, x_buf_iter + and x_tmp, x_tmp, 255 + ldr x_tmp, [x_crc_tab_addr, x_tmp, lsl 3] + eor x_seed, x_tmp, x_seed, lsr 8 + bne .loop_crc_tab + +.done: + mvn x_crc_ret, x_seed + ret + + .align 2 +.crc_clmul_pre: + fmov d_x0, x_seed // save crc to d_x0 + + crc_refl_load_first_block + + bls .clmul_loop_end + + crc64_load_p4 + +// 1024bit --> 512bit loop +// merge x0, x1, x2, x3, y0, y1, y2, y3 => x0, x1, x2, x3 (uint64x2_t) + crc_refl_loop + +.clmul_loop_end: +// folding 512bit --> 128bit + crc64_fold_512b_to_128b + +// folding 128bit --> 64bit + mov x_tmp, p0_low_b0 + movk x_tmp, p0_low_b1, lsl 16 + movk x_tmp, p0_low_b2, lsl 32 + movk x_tmp, p0_low_b3, lsl 48 + fmov d_p0_low, x_tmp + + pmull v_tmp_low.1q, v_x3.1d, v_p0.1d + + mov d_tmp_high, v_x3.d[1] + + eor v_x3.16b, v_tmp_high.16b, v_tmp_low.16b + +// barrett reduction + mov x_tmp, br_low_b0 + movk x_tmp, br_low_b1, lsl 16 + movk x_tmp, br_low_b2, lsl 32 + movk x_tmp, br_low_b3, lsl 48 + fmov d_br_low, x_tmp + + mov x_tmp2, br_high_b0 + movk x_tmp2, br_high_b1, lsl 16 + movk x_tmp2, br_high_b2, lsl 32 + movk x_tmp2, br_high_b3, lsl 48 + fmov d_br_high, x_tmp2 + + pmull v_tmp_low.1q, v_x3.1d, v_br_low.1d + pmull v_tmp_high.1q, v_tmp_low.1d, v_br_high.1d + + ext v_tmp_low.16b, v_br_low.16b, v_tmp_low.16b, #8 + + eor v_tmp_low.16b, v_tmp_low.16b, v_tmp_high.16b + eor v_tmp_low.16b, v_tmp_low.16b, v_x3.16b + umov x_crc_ret, v_tmp_low.d[1] + + b .crc_tab_pre + + .size \name, .-\name +.endm diff --git a/src/isa-l/crc/aarch64/crc_aarch64_dispatcher.c b/src/isa-l/crc/aarch64/crc_aarch64_dispatcher.c new file mode 100644 index 000000000..22ea72b14 --- /dev/null +++ b/src/isa-l/crc/aarch64/crc_aarch64_dispatcher.c @@ -0,0 +1,166 @@ +/********************************************************************** + Copyright(c) 2019-2020 Arm Corporation All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Arm Corporation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +**********************************************************************/ +#include <aarch64_multibinary.h> + +DEFINE_INTERFACE_DISPATCHER(crc16_t10dif) +{ + unsigned long auxval = getauxval(AT_HWCAP); + if (auxval & HWCAP_PMULL) + return PROVIDER_INFO(crc16_t10dif_pmull); + + return PROVIDER_BASIC(crc16_t10dif); + +} + +DEFINE_INTERFACE_DISPATCHER(crc16_t10dif_copy) +{ + unsigned long auxval = getauxval(AT_HWCAP); + if (auxval & HWCAP_PMULL) + return PROVIDER_INFO(crc16_t10dif_copy_pmull); + + return PROVIDER_BASIC(crc16_t10dif_copy); + +} + +DEFINE_INTERFACE_DISPATCHER(crc32_ieee) +{ + unsigned long auxval = getauxval(AT_HWCAP); + if (auxval & HWCAP_PMULL) { + return PROVIDER_INFO(crc32_ieee_norm_pmull); + } + + return PROVIDER_BASIC(crc32_ieee); + +} + +DEFINE_INTERFACE_DISPATCHER(crc32_iscsi) +{ + unsigned long auxval = getauxval(AT_HWCAP); + if (auxval & HWCAP_CRC32) { + switch (get_micro_arch_id()) { + case MICRO_ARCH_ID(ARM, NEOVERSE_N1): + case MICRO_ARCH_ID(ARM, CORTEX_A57): + case MICRO_ARCH_ID(ARM, CORTEX_A72): + return PROVIDER_INFO(crc32_iscsi_crc_ext); + } + } + if ((HWCAP_CRC32 | HWCAP_PMULL) == (auxval & (HWCAP_CRC32 | HWCAP_PMULL))) { + return PROVIDER_INFO(crc32_iscsi_3crc_fold); + } + + if (auxval & HWCAP_PMULL) { + return PROVIDER_INFO(crc32_iscsi_refl_pmull); + } + return PROVIDER_BASIC(crc32_iscsi); + +} + +DEFINE_INTERFACE_DISPATCHER(crc32_gzip_refl) +{ + unsigned long auxval = getauxval(AT_HWCAP); + + if (auxval & HWCAP_CRC32) { + switch (get_micro_arch_id()) { + case MICRO_ARCH_ID(ARM, NEOVERSE_N1): + case MICRO_ARCH_ID(ARM, CORTEX_A57): + case MICRO_ARCH_ID(ARM, CORTEX_A72): + return PROVIDER_INFO(crc32_gzip_refl_crc_ext); + } + } + if ((HWCAP_CRC32 | HWCAP_PMULL) == (auxval & (HWCAP_CRC32 | HWCAP_PMULL))) { + return PROVIDER_INFO(crc32_gzip_refl_3crc_fold); + } + + if (auxval & HWCAP_PMULL) + return PROVIDER_INFO(crc32_gzip_refl_pmull); + + return PROVIDER_BASIC(crc32_gzip_refl); + +} + +DEFINE_INTERFACE_DISPATCHER(crc64_ecma_refl) +{ + unsigned long auxval = getauxval(AT_HWCAP); + + if (auxval & HWCAP_PMULL) + return PROVIDER_INFO(crc64_ecma_refl_pmull); + + return PROVIDER_BASIC(crc64_ecma_refl); + +} + +DEFINE_INTERFACE_DISPATCHER(crc64_ecma_norm) +{ + unsigned long auxval = getauxval(AT_HWCAP); + if (auxval & HWCAP_PMULL) + return PROVIDER_INFO(crc64_ecma_norm_pmull); + + return PROVIDER_BASIC(crc64_ecma_norm); + +} + +DEFINE_INTERFACE_DISPATCHER(crc64_iso_refl) +{ + unsigned long auxval = getauxval(AT_HWCAP); + if (auxval & HWCAP_PMULL) + return PROVIDER_INFO(crc64_iso_refl_pmull); + + return PROVIDER_BASIC(crc64_iso_refl); + +} + +DEFINE_INTERFACE_DISPATCHER(crc64_iso_norm) +{ + unsigned long auxval = getauxval(AT_HWCAP); + if (auxval & HWCAP_PMULL) + return PROVIDER_INFO(crc64_iso_norm_pmull); + + return PROVIDER_BASIC(crc64_iso_norm); + +} + +DEFINE_INTERFACE_DISPATCHER(crc64_jones_refl) +{ + unsigned long auxval = getauxval(AT_HWCAP); + if (auxval & HWCAP_PMULL) + return PROVIDER_INFO(crc64_jones_refl_pmull); + + return PROVIDER_BASIC(crc64_jones_refl); + +} + +DEFINE_INTERFACE_DISPATCHER(crc64_jones_norm) +{ + unsigned long auxval = getauxval(AT_HWCAP); + if (auxval & HWCAP_PMULL) + return PROVIDER_INFO(crc64_jones_norm_pmull); + + return PROVIDER_BASIC(crc64_jones_norm); + +} diff --git a/src/isa-l/crc/aarch64/crc_common_pmull.h b/src/isa-l/crc/aarch64/crc_common_pmull.h new file mode 100644 index 000000000..20a71b913 --- /dev/null +++ b/src/isa-l/crc/aarch64/crc_common_pmull.h @@ -0,0 +1,302 @@ +######################################################################## +# Copyright (c) 2019 Microsoft Corporation. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Microsoft Corporation nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +######################################################################### + +// parameters +#define w_seed w0 +#define x_seed x0 +#define x_buf x1 +#define w_len w2 +#define x_len x2 + +// return +#define w_crc_ret w0 +#define x_crc_ret x0 + +// constant +#define FOLD_SIZE 64 + +// global variables +#define x_buf_end x3 +#define w_counter w4 +#define x_counter x4 +#define x_buf_iter x5 +#define x_crc_tab_addr x6 +#define x_tmp2 x6 +#define w_tmp w7 +#define x_tmp x7 + +#define v_x0 v0 +#define d_x0 d0 +#define s_x0 s0 + +#define q_x1 q1 +#define v_x1 v1 + +#define q_x2 q2 +#define v_x2 v2 + +#define q_x3 q3 +#define v_x3 v3 +#define d_x3 d3 +#define s_x3 s3 + +#define q_y0 q4 +#define v_y0 v4 +#define v_tmp_high v4 +#define d_tmp_high d4 + +#define q_y1 q5 +#define v_y1 v5 +#define v_tmp_low v5 + +#define q_y2 q6 +#define v_y2 v6 + +#define q_y3 q7 +#define v_y3 v7 + +#define q_x0_tmp q30 +#define v_x0_tmp v30 +#define d_p4_high v30.d[1] +#define d_p4_low d30 +#define v_p4 v30 +#define d_p1_high v30.d[1] +#define d_p1_low d30 +#define v_p1 v30 +#define d_p0_high v30.d[1] +#define d_p0_low d30 +#define v_p0 v30 +#define d_br_low d30 +#define d_br_low2 v30.d[1] +#define v_br_low v30 + +#define q_shuffle q31 +#define v_shuffle v31 +#define d_br_high d31 +#define d_br_high2 v31.d[1] +#define v_br_high v31 +#define d_p0_low2 d31 +#define d_p0_high2 v31.d[1] +#define v_p02 v31 + +#define v_x0_high v16 +#define v_x1_high v17 +#define v_x2_high v18 +#define v_x3_high v19 + +.macro crc_refl_load_first_block + ldr q_x0_tmp, [x_buf] + ldr q_x1, [x_buf, 16] + ldr q_x2, [x_buf, 32] + ldr q_x3, [x_buf, 48] + + and x_counter, x_len, -64 + sub x_tmp, x_counter, #64 + cmp x_tmp, 63 + + add x_buf_iter, x_buf, 64 + + eor v_x0.16b, v_x0.16b, v_x0_tmp.16b +.endm + +.macro crc_norm_load_first_block + adrp x_tmp, .shuffle_data + ldr q_shuffle, [x_tmp, #:lo12:.shuffle_data] + + ldr q_x0_tmp, [x_buf] + ldr q_x1, [x_buf, 16] + ldr q_x2, [x_buf, 32] + ldr q_x3, [x_buf, 48] + + and x_counter, x_len, -64 + sub x_tmp, x_counter, #64 + cmp x_tmp, 63 + + add x_buf_iter, x_buf, 64 + + tbl v_x0_tmp.16b, {v_x0_tmp.16b}, v_shuffle.16b + tbl v_x1.16b, {v_x1.16b}, v_shuffle.16b + tbl v_x2.16b, {v_x2.16b}, v_shuffle.16b + tbl v_x3.16b, {v_x3.16b}, v_shuffle.16b + + eor v_x0.16b, v_x0.16b, v_x0_tmp.16b +.endm + +.macro crc32_load_p4 + add x_buf_end, x_buf_iter, x_tmp + + mov x_tmp, p4_low_b0 + movk x_tmp, p4_low_b1, lsl 16 + fmov d_p4_low, x_tmp + + mov x_tmp2, p4_high_b0 + movk x_tmp2, p4_high_b1, lsl 16 + fmov d_p4_high, x_tmp2 +.endm + +.macro crc64_load_p4 + add x_buf_end, x_buf_iter, x_tmp + + mov x_tmp, p4_low_b0 + movk x_tmp, p4_low_b1, lsl 16 + movk x_tmp, p4_low_b2, lsl 32 + movk x_tmp, p4_low_b3, lsl 48 + fmov d_p4_low, x_tmp + + mov x_tmp2, p4_high_b0 + movk x_tmp2, p4_high_b1, lsl 16 + movk x_tmp2, p4_high_b2, lsl 32 + movk x_tmp2, p4_high_b3, lsl 48 + fmov d_p4_high, x_tmp2 +.endm + +.macro crc_refl_loop + .align 3 +.clmul_loop: + // interleave ldr and pmull(2) for arch which can only issue quadword load every + // other cycle (i.e. A55) + ldr q_y0, [x_buf_iter] + pmull2 v_x0_high.1q, v_x0.2d, v_p4.2d + ldr q_y1, [x_buf_iter, 16] + pmull2 v_x1_high.1q, v_x1.2d, v_p4.2d + ldr q_y2, [x_buf_iter, 32] + pmull2 v_x2_high.1q, v_x2.2d, v_p4.2d + ldr q_y3, [x_buf_iter, 48] + pmull2 v_x3_high.1q, v_x3.2d, v_p4.2d + + pmull v_x0.1q, v_x0.1d, v_p4.1d + add x_buf_iter, x_buf_iter, 64 + pmull v_x1.1q, v_x1.1d, v_p4.1d + cmp x_buf_iter, x_buf_end + pmull v_x2.1q, v_x2.1d, v_p4.1d + pmull v_x3.1q, v_x3.1d, v_p4.1d + + eor v_x0.16b, v_x0.16b, v_x0_high.16b + eor v_x1.16b, v_x1.16b, v_x1_high.16b + eor v_x2.16b, v_x2.16b, v_x2_high.16b + eor v_x3.16b, v_x3.16b, v_x3_high.16b + + eor v_x0.16b, v_x0.16b, v_y0.16b + eor v_x1.16b, v_x1.16b, v_y1.16b + eor v_x2.16b, v_x2.16b, v_y2.16b + eor v_x3.16b, v_x3.16b, v_y3.16b + bne .clmul_loop +.endm + +.macro crc_norm_loop + .align 3 +.clmul_loop: + // interleave ldr and pmull(2) for arch which can only issue quadword load every + // other cycle (i.e. A55) + ldr q_y0, [x_buf_iter] + pmull2 v_x0_high.1q, v_x0.2d, v_p4.2d + ldr q_y1, [x_buf_iter, 16] + pmull2 v_x1_high.1q, v_x1.2d, v_p4.2d + ldr q_y2, [x_buf_iter, 32] + pmull2 v_x2_high.1q, v_x2.2d, v_p4.2d + ldr q_y3, [x_buf_iter, 48] + pmull2 v_x3_high.1q, v_x3.2d, v_p4.2d + + pmull v_x0.1q, v_x0.1d, v_p4.1d + add x_buf_iter, x_buf_iter, 64 + pmull v_x1.1q, v_x1.1d, v_p4.1d + cmp x_buf_iter, x_buf_end + pmull v_x2.1q, v_x2.1d, v_p4.1d + pmull v_x3.1q, v_x3.1d, v_p4.1d + + tbl v_y0.16b, {v_y0.16b}, v_shuffle.16b + tbl v_y1.16b, {v_y1.16b}, v_shuffle.16b + tbl v_y2.16b, {v_y2.16b}, v_shuffle.16b + tbl v_y3.16b, {v_y3.16b}, v_shuffle.16b + + eor v_x0.16b, v_x0.16b, v_x0_high.16b + eor v_x1.16b, v_x1.16b, v_x1_high.16b + eor v_x2.16b, v_x2.16b, v_x2_high.16b + eor v_x3.16b, v_x3.16b, v_x3_high.16b + + eor v_x0.16b, v_x0.16b, v_y0.16b + eor v_x1.16b, v_x1.16b, v_y1.16b + eor v_x2.16b, v_x2.16b, v_y2.16b + eor v_x3.16b, v_x3.16b, v_y3.16b + bne .clmul_loop +.endm + +.macro crc32_fold_512b_to_128b + mov x_tmp, p1_low_b0 + movk x_tmp, p1_low_b1, lsl 16 + fmov d_p1_low, x_tmp + + mov x_tmp2, p1_high_b0 + movk x_tmp2, p1_high_b1, lsl 16 + fmov d_p1_high, x_tmp2 + + pmull2 v_tmp_high.1q, v_x0.2d, v_p1.2d + pmull v_tmp_low.1q, v_x0.1d, v_p1.1d + eor v_x1.16b, v_x1.16b, v_tmp_high.16b + eor v_x1.16b, v_x1.16b, v_tmp_low.16b + + pmull2 v_tmp_high.1q, v_x1.2d, v_p1.2d + pmull v_tmp_low.1q, v_x1.1d, v_p1.1d + eor v_x2.16b, v_x2.16b, v_tmp_high.16b + eor v_x2.16b, v_x2.16b, v_tmp_low.16b + + pmull2 v_tmp_high.1q, v_x2.2d, v_p1.2d + pmull v_tmp_low.1q, v_x2.1d, v_p1.1d + eor v_x3.16b, v_x3.16b, v_tmp_high.16b + eor v_x3.16b, v_x3.16b, v_tmp_low.16b +.endm + +.macro crc64_fold_512b_to_128b + mov x_tmp, p1_low_b0 + movk x_tmp, p1_low_b1, lsl 16 + movk x_tmp, p1_low_b2, lsl 32 + movk x_tmp, p1_low_b3, lsl 48 + fmov d_p1_low, x_tmp + + mov x_tmp2, p1_high_b0 + movk x_tmp2, p1_high_b1, lsl 16 + movk x_tmp2, p1_high_b2, lsl 32 + movk x_tmp2, p1_high_b3, lsl 48 + fmov d_p1_high, x_tmp2 + + pmull2 v_tmp_high.1q, v_x0.2d, v_p1.2d + pmull v_tmp_low.1q, v_x0.1d, v_p1.1d + eor v_x1.16b, v_x1.16b, v_tmp_high.16b + eor v_x1.16b, v_x1.16b, v_tmp_low.16b + + pmull2 v_tmp_high.1q, v_x1.2d, v_p1.2d + pmull v_tmp_low.1q, v_x1.1d, v_p1.1d + eor v_x2.16b, v_x2.16b, v_tmp_high.16b + eor v_x2.16b, v_x2.16b, v_tmp_low.16b + + pmull2 v_tmp_high.1q, v_x2.2d, v_p1.2d + pmull v_tmp_low.1q, v_x2.1d, v_p1.1d + eor v_x3.16b, v_x3.16b, v_tmp_high.16b + eor v_x3.16b, v_x3.16b, v_tmp_low.16b +.endm
\ No newline at end of file diff --git a/src/isa-l/crc/aarch64/crc_multibinary_arm.S b/src/isa-l/crc/aarch64/crc_multibinary_arm.S new file mode 100644 index 000000000..76f957164 --- /dev/null +++ b/src/isa-l/crc/aarch64/crc_multibinary_arm.S @@ -0,0 +1,42 @@ +######################################################################## +# Copyright(c) 2019 Arm Corporation All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Arm Corporation nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +######################################################################### +#include <aarch64_multibinary.h> + + +mbin_interface crc32_iscsi +mbin_interface crc16_t10dif +mbin_interface crc16_t10dif_copy +mbin_interface crc32_ieee +mbin_interface crc32_gzip_refl +mbin_interface crc64_ecma_refl +mbin_interface crc64_ecma_norm +mbin_interface crc64_iso_refl +mbin_interface crc64_iso_norm +mbin_interface crc64_jones_refl +mbin_interface crc64_jones_norm |