/* * Copyright © 2018, VideoLAN and dav1d authors * Copyright © 2018, Janne Grunau * Copyright © 2020, Martin Storsjo * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, this * list of conditions and the following disclaimer. * * 2. Redistributions in binary form must reproduce the above copyright notice, * this list of conditions and the following disclaimer in the documentation * and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include "src/arm/asm.S" #include "util.S" #define PREP_BIAS 8192 .macro avg d0, d1, t0, t1, t2, t3 ld1 {\t0\().8h,\t1\().8h}, [x2], 32 ld1 {\t2\().8h,\t3\().8h}, [x3], 32 sqadd \t0\().8h, \t0\().8h, \t2\().8h sqadd \t1\().8h, \t1\().8h, \t3\().8h smax \t0\().8h, \t0\().8h, v28.8h // -2*PREP_BIAS - 1 << intermediate_bits smax \t1\().8h, \t1\().8h, v28.8h // -2*PREP_BIAS - 1 << intermediate_bits sqsub \t0\().8h, \t0\().8h, v28.8h // -2*PREP_BIAS - 1 << intermediate_bits sqsub \t1\().8h, \t1\().8h, v28.8h // -2*PREP_BIAS - 1 << intermediate_bits sshl \d0\().8h, \t0\().8h, v29.8h // -(intermediate_bits+1) sshl \d1\().8h, \t1\().8h, v29.8h // -(intermediate_bits+1) .endm .macro w_avg d0, d1, t0, t1, t2, t3 ld1 {\t0\().8h,\t1\().8h}, [x2], 32 ld1 {\t2\().8h,\t3\().8h}, [x3], 32 // This difference requires a 17 bit range, and all bits are // significant for the following multiplication. ssubl \d0\().4s, \t2\().4h, \t0\().4h ssubl2 \t0\().4s, \t2\().8h, \t0\().8h ssubl \d1\().4s, \t3\().4h, \t1\().4h ssubl2 \t1\().4s, \t3\().8h, \t1\().8h mul \d0\().4s, \d0\().4s, v27.4s mul \t0\().4s, \t0\().4s, v27.4s mul \d1\().4s, \d1\().4s, v27.4s mul \t1\().4s, \t1\().4s, v27.4s sshr \d0\().4s, \d0\().4s, #4 sshr \t0\().4s, \t0\().4s, #4 sshr \d1\().4s, \d1\().4s, #4 sshr \t1\().4s, \t1\().4s, #4 saddw \d0\().4s, \d0\().4s, \t2\().4h saddw2 \t0\().4s, \t0\().4s, \t2\().8h saddw \d1\().4s, \d1\().4s, \t3\().4h saddw2 \t1\().4s, \t1\().4s, \t3\().8h uzp1 \d0\().8h, \d0\().8h, \t0\().8h // Same as xtn, xtn2 uzp1 \d1\().8h, \d1\().8h, \t1\().8h // Ditto srshl \d0\().8h, \d0\().8h, v29.8h // -intermediate_bits srshl \d1\().8h, \d1\().8h, v29.8h // -intermediate_bits add \d0\().8h, \d0\().8h, v28.8h // PREP_BIAS >> intermediate_bits add \d1\().8h, \d1\().8h, v28.8h // PREP_BIAS >> intermediate_bits smin \d0\().8h, \d0\().8h, v31.8h // bitdepth_max smin \d1\().8h, \d1\().8h, v31.8h // bitdepth_max smax \d0\().8h, \d0\().8h, v30.8h // 0 smax \d1\().8h, \d1\().8h, v30.8h // 0 .endm .macro mask d0, d1, t0, t1, t2, t3 ld1 {v27.16b}, [x6], 16 ld1 {\t0\().8h,\t1\().8h}, [x2], 32 neg v27.16b, v27.16b ld1 {\t2\().8h,\t3\().8h}, [x3], 32 sxtl v26.8h, v27.8b sxtl2 v27.8h, v27.16b sxtl v24.4s, v26.4h sxtl2 v25.4s, v26.8h sxtl v26.4s, v27.4h sxtl2 v27.4s, v27.8h ssubl \d0\().4s, \t2\().4h, \t0\().4h ssubl2 \t0\().4s, \t2\().8h, \t0\().8h ssubl \d1\().4s, \t3\().4h, \t1\().4h ssubl2 \t1\().4s, \t3\().8h, \t1\().8h mul \d0\().4s, \d0\().4s, v24.4s mul \t0\().4s, \t0\().4s, v25.4s mul \d1\().4s, \d1\().4s, v26.4s mul \t1\().4s, \t1\().4s, v27.4s sshr \d0\().4s, \d0\().4s, #6 sshr \t0\().4s, \t0\().4s, #6 sshr \d1\().4s, \d1\().4s, #6 sshr \t1\().4s, \t1\().4s, #6 saddw \d0\().4s, \d0\().4s, \t2\().4h saddw2 \t0\().4s, \t0\().4s, \t2\().8h saddw \d1\().4s, \d1\().4s, \t3\().4h saddw2 \t1\().4s, \t1\().4s, \t3\().8h uzp1 \d0\().8h, \d0\().8h, \t0\().8h // Same as xtn, xtn2 uzp1 \d1\().8h, \d1\().8h, \t1\().8h // Ditto srshl \d0\().8h, \d0\().8h, v29.8h // -intermediate_bits srshl \d1\().8h, \d1\().8h, v29.8h // -intermediate_bits add \d0\().8h, \d0\().8h, v28.8h // PREP_BIAS >> intermediate_bits add \d1\().8h, \d1\().8h, v28.8h // PREP_BIAS >> intermediate_bits smin \d0\().8h, \d0\().8h, v31.8h // bitdepth_max smin \d1\().8h, \d1\().8h, v31.8h // bitdepth_max smax \d0\().8h, \d0\().8h, v30.8h // 0 smax \d1\().8h, \d1\().8h, v30.8h // 0 .endm .macro bidir_fn type, bdmax function \type\()_16bpc_neon, export=1 clz w4, w4 .ifnc \type, avg dup v31.8h, \bdmax // bitdepth_max movi v30.8h, #0 .endif clz w7, \bdmax sub w7, w7, #18 // intermediate_bits = clz(bitdepth_max) - 18 .ifc \type, avg mov w9, #1 mov w8, #-2*PREP_BIAS lsl w9, w9, w7 // 1 << intermediate_bits add w7, w7, #1 sub w8, w8, w9 // -2*PREP_BIAS - 1 << intermediate_bits neg w7, w7 // -(intermediate_bits+1) dup v28.8h, w8 // -2*PREP_BIAS - 1 << intermediate_bits dup v29.8h, w7 // -(intermediate_bits+1) .else mov w8, #PREP_BIAS lsr w8, w8, w7 // PREP_BIAS >> intermediate_bits neg w7, w7 // -intermediate_bits dup v28.8h, w8 // PREP_BIAS >> intermediate_bits dup v29.8h, w7 // -intermediate_bits .endif .ifc \type, w_avg dup v27.4s, w6 neg v27.4s, v27.4s .endif adr x7, L(\type\()_tbl) sub w4, w4, #24 \type v4, v5, v0, v1, v2, v3 ldrh w4, [x7, x4, lsl #1] sub x7, x7, w4, uxtw br x7 40: AARCH64_VALID_JUMP_TARGET add x7, x0, x1 lsl x1, x1, #1 4: subs w5, w5, #4 st1 {v4.d}[0], [x0], x1 st1 {v4.d}[1], [x7], x1 st1 {v5.d}[0], [x0], x1 st1 {v5.d}[1], [x7], x1 b.le 0f \type v4, v5, v0, v1, v2, v3 b 4b 80: AARCH64_VALID_JUMP_TARGET add x7, x0, x1 lsl x1, x1, #1 8: st1 {v4.8h}, [x0], x1 subs w5, w5, #2 st1 {v5.8h}, [x7], x1 b.le 0f \type v4, v5, v0, v1, v2, v3 b 8b 16: AARCH64_VALID_JUMP_TARGET \type v6, v7, v0, v1, v2, v3 st1 {v4.8h, v5.8h}, [x0], x1 subs w5, w5, #2 st1 {v6.8h, v7.8h}, [x0], x1 b.le 0f \type v4, v5, v0, v1, v2, v3 b 16b 32: AARCH64_VALID_JUMP_TARGET \type v6, v7, v0, v1, v2, v3 subs w5, w5, #1 st1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x0], x1 b.le 0f \type v4, v5, v0, v1, v2, v3 b 32b 640: AARCH64_VALID_JUMP_TARGET add x7, x0, #64 64: \type v6, v7, v0, v1, v2, v3 \type v16, v17, v0, v1, v2, v3 st1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x0], x1 \type v18, v19, v0, v1, v2, v3 subs w5, w5, #1 st1 {v16.8h,v17.8h,v18.8h,v19.8h}, [x7], x1 b.le 0f \type v4, v5, v0, v1, v2, v3 b 64b 1280: AARCH64_VALID_JUMP_TARGET add x7, x0, #64 mov x8, #128 sub x1, x1, #128 128: \type v6, v7, v0, v1, v2, v3 \type v16, v17, v0, v1, v2, v3 st1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x0], x8 \type v18, v19, v0, v1, v2, v3 st1 {v16.8h,v17.8h,v18.8h,v19.8h}, [x7], x8 \type v4, v5, v0, v1, v2, v3 \type v6, v7, v0, v1, v2, v3 \type v16, v17, v0, v1, v2, v3 subs w5, w5, #1 st1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x0], x1 \type v18, v19, v0, v1, v2, v3 st1 {v16.8h,v17.8h,v18.8h,v19.8h}, [x7], x1 b.le 0f \type v4, v5, v0, v1, v2, v3 b 128b 0: ret L(\type\()_tbl): .hword L(\type\()_tbl) - 1280b .hword L(\type\()_tbl) - 640b .hword L(\type\()_tbl) - 32b .hword L(\type\()_tbl) - 16b .hword L(\type\()_tbl) - 80b .hword L(\type\()_tbl) - 40b endfunc .endm bidir_fn avg, w6 bidir_fn w_avg, w7 bidir_fn mask, w7 .macro w_mask_fn type function w_mask_\type\()_16bpc_neon, export=1 ldr w8, [sp] clz w9, w4 adr x10, L(w_mask_\type\()_tbl) dup v31.8h, w8 // bitdepth_max sub w9, w9, #24 clz w8, w8 // clz(bitdepth_max) ldrh w9, [x10, x9, lsl #1] sub x10, x10, w9, uxtw sub w8, w8, #12 // sh = intermediate_bits + 6 = clz(bitdepth_max) - 12 mov w9, #PREP_BIAS*64 neg w8, w8 // -sh mov w11, #27615 // (64 + 1 - 38)<> mask_sh ushr v21.8h, v21.8h, #10 add v4.4s, v4.4s, v30.4s // += PREP_BIAS*64 add v5.4s, v5.4s, v30.4s add v6.4s, v6.4s, v30.4s add v7.4s, v7.4s, v30.4s uxtl v22.4s, v20.4h uxtl2 v23.4s, v20.8h uxtl v24.4s, v21.4h uxtl2 v25.4s, v21.8h mla v4.4s, v16.4s, v22.4s // (tmp2-tmp1)*(64-m) mla v5.4s, v17.4s, v23.4s mla v6.4s, v18.4s, v24.4s mla v7.4s, v19.4s, v25.4s srshl v4.4s, v4.4s, v29.4s // (tmp1<<6 + (tmp2-tmp1)*(64-m) + (1 << (sh-1)) + PREP_BIAS*64) >> sh srshl v5.4s, v5.4s, v29.4s srshl v6.4s, v6.4s, v29.4s srshl v7.4s, v7.4s, v29.4s sqxtun v4.4h, v4.4s // iclip_pixel sqxtun2 v4.8h, v5.4s sqxtun v5.4h, v6.4s sqxtun2 v5.8h, v7.4s umin v4.8h, v4.8h, v31.8h // iclip_pixel umin v5.8h, v5.8h, v31.8h .if \type == 444 uzp1 v20.16b, v20.16b, v21.16b // 64 - m sub v20.16b, v1.16b, v20.16b // m st1 {v20.16b}, [x6], #16 .elseif \type == 422 addp v20.8h, v20.8h, v21.8h // (64 - m) + (64 - n) (column wise addition) xtn v20.8b, v20.8h uhsub v20.8b, v3.8b, v20.8b // ((129 - sign) - ((64 - m) + (64 - n)) >> 1 st1 {v20.8b}, [x6], #8 .elseif \type == 420 trn1 v24.2d, v20.2d, v21.2d trn2 v25.2d, v20.2d, v21.2d add v24.8h, v24.8h, v25.8h // (64 - my1) + (64 - my2) (row wise addition) addp v20.8h, v24.8h, v24.8h // (128 - m) + (128 - n) (column wise addition) sub v20.4h, v3.4h, v20.4h // (256 - sign) - ((128 - m) + (128 - n)) rshrn v20.8b, v20.8h, #2 // ((256 - sign) - ((128 - m) + (128 - n)) + 2) >> 2 st1 {v20.s}[0], [x6], #4 .endif st1 {v4.d}[0], [x0], x1 st1 {v4.d}[1], [x12], x1 st1 {v5.d}[0], [x0], x1 st1 {v5.d}[1], [x12], x1 b.gt 4b ret 8: AARCH64_VALID_JUMP_TARGET ld1 {v4.8h, v5.8h}, [x2], #32 // tmp1 ld1 {v6.8h, v7.8h}, [x3], #32 // tmp2 subs w5, w5, #2 sabd v20.8h, v4.8h, v6.8h // abs(tmp1 - tmp2) sabd v21.8h, v5.8h, v7.8h ssubl v16.4s, v6.4h, v4.4h // tmp2 - tmp1 (requires 17 bit) ssubl2 v17.4s, v6.8h, v4.8h ssubl v18.4s, v7.4h, v5.4h ssubl2 v19.4s, v7.8h, v5.8h uqsub v20.8h, v0.8h, v20.8h // 27615 - abs() uqsub v21.8h, v0.8h, v21.8h sshll2 v7.4s, v5.8h, #6 // tmp1 << 6 sshll v6.4s, v5.4h, #6 sshll2 v5.4s, v4.8h, #6 sshll v4.4s, v4.4h, #6 ushr v20.8h, v20.8h, #10 // 64-m = (27615 - abs()) >> mask_sh ushr v21.8h, v21.8h, #10 add v4.4s, v4.4s, v30.4s // += PREP_BIAS*64 add v5.4s, v5.4s, v30.4s add v6.4s, v6.4s, v30.4s add v7.4s, v7.4s, v30.4s uxtl v22.4s, v20.4h uxtl2 v23.4s, v20.8h uxtl v24.4s, v21.4h uxtl2 v25.4s, v21.8h mla v4.4s, v16.4s, v22.4s // (tmp2-tmp1)*(64-m) mla v5.4s, v17.4s, v23.4s mla v6.4s, v18.4s, v24.4s mla v7.4s, v19.4s, v25.4s srshl v4.4s, v4.4s, v29.4s // (tmp1<<6 + (tmp2-tmp1)*(64-m) + (1 << (sh-1)) + PREP_BIAS*64) >> sh srshl v5.4s, v5.4s, v29.4s srshl v6.4s, v6.4s, v29.4s srshl v7.4s, v7.4s, v29.4s sqxtun v4.4h, v4.4s // iclip_pixel sqxtun2 v4.8h, v5.4s sqxtun v5.4h, v6.4s sqxtun2 v5.8h, v7.4s umin v4.8h, v4.8h, v31.8h // iclip_pixel umin v5.8h, v5.8h, v31.8h .if \type == 444 uzp1 v20.16b, v20.16b, v21.16b // 64 - m sub v20.16b, v1.16b, v20.16b // m st1 {v20.16b}, [x6], #16 .elseif \type == 422 addp v20.8h, v20.8h, v21.8h // (64 - m) + (64 - n) (column wise addition) xtn v20.8b, v20.8h uhsub v20.8b, v3.8b, v20.8b // ((129 - sign) - ((64 - m) + (64 - n)) >> 1 st1 {v20.8b}, [x6], #8 .elseif \type == 420 add v20.8h, v20.8h, v21.8h // (64 - my1) + (64 - my2) (row wise addition) addp v20.8h, v20.8h, v20.8h // (128 - m) + (128 - n) (column wise addition) sub v20.4h, v3.4h, v20.4h // (256 - sign) - ((128 - m) + (128 - n)) rshrn v20.8b, v20.8h, #2 // ((256 - sign) - ((128 - m) + (128 - n)) + 2) >> 2 st1 {v20.s}[0], [x6], #4 .endif st1 {v4.8h}, [x0], x1 st1 {v5.8h}, [x12], x1 b.gt 8b ret 1280: 640: 320: 160: AARCH64_VALID_JUMP_TARGET mov w11, w4 sub x1, x1, w4, uxtw #1 .if \type == 444 add x10, x6, w4, uxtw .elseif \type == 422 add x10, x6, x11, lsr #1 .endif add x9, x3, w4, uxtw #1 add x7, x2, w4, uxtw #1 161: mov w8, w4 16: ld1 {v4.8h, v5.8h}, [x2], #32 // tmp1 ld1 {v16.8h, v17.8h}, [x3], #32 // tmp2 ld1 {v6.8h, v7.8h}, [x7], #32 ld1 {v18.8h, v19.8h}, [x9], #32 subs w8, w8, #16 sabd v20.8h, v4.8h, v16.8h // abs(tmp1 - tmp2) sabd v21.8h, v5.8h, v17.8h ssubl v22.4s, v16.4h, v4.4h // tmp2 - tmp1 (requires 17 bit) ssubl2 v23.4s, v16.8h, v4.8h ssubl v24.4s, v17.4h, v5.4h ssubl2 v25.4s, v17.8h, v5.8h uqsub v20.8h, v0.8h, v20.8h // 27615 - abs() uqsub v21.8h, v0.8h, v21.8h sshll2 v27.4s, v5.8h, #6 // tmp1 << 6 sshll v26.4s, v5.4h, #6 sshll2 v5.4s, v4.8h, #6 sshll v4.4s, v4.4h, #6 ushr v20.8h, v20.8h, #10 // 64-m = (27615 - abs()) >> mask_sh ushr v21.8h, v21.8h, #10 add v4.4s, v4.4s, v30.4s // += PREP_BIAS*64 add v5.4s, v5.4s, v30.4s add v26.4s, v26.4s, v30.4s add v27.4s, v27.4s, v30.4s uxtl v16.4s, v20.4h uxtl2 v17.4s, v20.8h uxtl v28.4s, v21.4h mla v4.4s, v22.4s, v16.4s // (tmp2-tmp1)*(64-m) uxtl2 v16.4s, v21.8h mla v5.4s, v23.4s, v17.4s mla v26.4s, v24.4s, v28.4s mla v27.4s, v25.4s, v16.4s srshl v4.4s, v4.4s, v29.4s // (tmp1<<6 + (tmp2-tmp1)*(64-m) + (1 << (sh-1)) + PREP_BIAS*64) >> sh srshl v5.4s, v5.4s, v29.4s srshl v26.4s, v26.4s, v29.4s srshl v27.4s, v27.4s, v29.4s sqxtun v4.4h, v4.4s // iclip_pixel sqxtun2 v4.8h, v5.4s sqxtun v5.4h, v26.4s sqxtun2 v5.8h, v27.4s // Start of other half sabd v22.8h, v6.8h, v18.8h // abs(tmp1 - tmp2) sabd v23.8h, v7.8h, v19.8h umin v4.8h, v4.8h, v31.8h // iclip_pixel umin v5.8h, v5.8h, v31.8h ssubl v16.4s, v18.4h, v6.4h // tmp2 - tmp1 (requires 17 bit) ssubl2 v17.4s, v18.8h, v6.8h ssubl v18.4s, v19.4h, v7.4h ssubl2 v19.4s, v19.8h, v7.8h uqsub v22.8h, v0.8h, v22.8h // 27615 - abs() uqsub v23.8h, v0.8h, v23.8h sshll v24.4s, v6.4h, #6 // tmp1 << 6 sshll2 v25.4s, v6.8h, #6 sshll v26.4s, v7.4h, #6 sshll2 v27.4s, v7.8h, #6 ushr v22.8h, v22.8h, #10 // 64-m = (27615 - abs()) >> mask_sh ushr v23.8h, v23.8h, #10 add v24.4s, v24.4s, v30.4s // += PREP_BIAS*64 add v25.4s, v25.4s, v30.4s add v26.4s, v26.4s, v30.4s add v27.4s, v27.4s, v30.4s uxtl v6.4s, v22.4h uxtl2 v7.4s, v22.8h uxtl v28.4s, v23.4h mla v24.4s, v16.4s, v6.4s // (tmp2-tmp1)*(64-m) uxtl2 v6.4s, v23.8h mla v25.4s, v17.4s, v7.4s mla v26.4s, v18.4s, v28.4s mla v27.4s, v19.4s, v6.4s srshl v24.4s, v24.4s, v29.4s // (tmp1<<6 + (tmp2-tmp1)*(64-m) + (1 << (sh-1)) + PREP_BIAS*64) >> sh srshl v25.4s, v25.4s, v29.4s srshl v26.4s, v26.4s, v29.4s srshl v27.4s, v27.4s, v29.4s sqxtun v6.4h, v24.4s // iclip_pixel sqxtun2 v6.8h, v25.4s sqxtun v7.4h, v26.4s sqxtun2 v7.8h, v27.4s umin v6.8h, v6.8h, v31.8h // iclip_pixel umin v7.8h, v7.8h, v31.8h .if \type == 444 uzp1 v20.16b, v20.16b, v21.16b // 64 - m uzp1 v21.16b, v22.16b, v23.16b sub v20.16b, v1.16b, v20.16b // m sub v21.16b, v1.16b, v21.16b st1 {v20.16b}, [x6], #16 st1 {v21.16b}, [x10], #16 .elseif \type == 422 addp v20.8h, v20.8h, v21.8h // (64 - m) + (64 - n) (column wise addition) addp v21.8h, v22.8h, v23.8h xtn v20.8b, v20.8h xtn v21.8b, v21.8h uhsub v20.8b, v3.8b, v20.8b // ((129 - sign) - ((64 - m) + (64 - n)) >> 1 uhsub v21.8b, v3.8b, v21.8b st1 {v20.8b}, [x6], #8 st1 {v21.8b}, [x10], #8 .elseif \type == 420 add v20.8h, v20.8h, v22.8h // (64 - my1) + (64 - my2) (row wise addition) add v21.8h, v21.8h, v23.8h addp v20.8h, v20.8h, v21.8h // (128 - m) + (128 - n) (column wise addition) sub v20.8h, v3.8h, v20.8h // (256 - sign) - ((128 - m) + (128 - n)) rshrn v20.8b, v20.8h, #2 // ((256 - sign) - ((128 - m) + (128 - n)) + 2) >> 2 st1 {v20.8b}, [x6], #8 .endif st1 {v4.8h, v5.8h}, [x0], #32 st1 {v6.8h, v7.8h}, [x12], #32 b.gt 16b subs w5, w5, #2 add x2, x2, w4, uxtw #1 add x3, x3, w4, uxtw #1 add x7, x7, w4, uxtw #1 add x9, x9, w4, uxtw #1 .if \type == 444 add x6, x6, w4, uxtw add x10, x10, w4, uxtw .elseif \type == 422 add x6, x6, x11, lsr #1 add x10, x10, x11, lsr #1 .endif add x0, x0, x1 add x12, x12, x1 b.gt 161b ret L(w_mask_\type\()_tbl): .hword L(w_mask_\type\()_tbl) - 1280b .hword L(w_mask_\type\()_tbl) - 640b .hword L(w_mask_\type\()_tbl) - 320b .hword L(w_mask_\type\()_tbl) - 160b .hword L(w_mask_\type\()_tbl) - 8b .hword L(w_mask_\type\()_tbl) - 4b endfunc .endm w_mask_fn 444 w_mask_fn 422 w_mask_fn 420 function blend_16bpc_neon, export=1 adr x6, L(blend_tbl) clz w3, w3 sub w3, w3, #26 ldrh w3, [x6, x3, lsl #1] sub x6, x6, w3, uxtw add x8, x0, x1 br x6 40: AARCH64_VALID_JUMP_TARGET lsl x1, x1, #1 4: ld1 {v2.8b}, [x5], #8 ld1 {v1.8h}, [x2], #16 ld1 {v0.d}[0], [x0] neg v2.8b, v2.8b // -m subs w4, w4, #2 ld1 {v0.d}[1], [x8] sxtl v2.8h, v2.8b shl v2.8h, v2.8h, #9 // -m << 9 sub v1.8h, v0.8h, v1.8h // a - b sqrdmulh v1.8h, v1.8h, v2.8h // ((a-b)*-m + 32) >> 6 add v0.8h, v0.8h, v1.8h st1 {v0.d}[0], [x0], x1 st1 {v0.d}[1], [x8], x1 b.gt 4b ret 80: AARCH64_VALID_JUMP_TARGET lsl x1, x1, #1 8: ld1 {v4.16b}, [x5], #16 ld1 {v2.8h, v3.8h}, [x2], #32 neg v5.16b, v4.16b // -m ld1 {v0.8h}, [x0] ld1 {v1.8h}, [x8] sxtl v4.8h, v5.8b sxtl2 v5.8h, v5.16b shl v4.8h, v4.8h, #9 // -m << 9 shl v5.8h, v5.8h, #9 sub v2.8h, v0.8h, v2.8h // a - b sub v3.8h, v1.8h, v3.8h subs w4, w4, #2 sqrdmulh v2.8h, v2.8h, v4.8h // ((a-b)*-m + 32) >> 6 sqrdmulh v3.8h, v3.8h, v5.8h add v0.8h, v0.8h, v2.8h add v1.8h, v1.8h, v3.8h st1 {v0.8h}, [x0], x1 st1 {v1.8h}, [x8], x1 b.gt 8b ret 160: AARCH64_VALID_JUMP_TARGET lsl x1, x1, #1 16: ld1 {v16.16b, v17.16b}, [x5], #32 ld1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x2], #64 subs w4, w4, #2 neg v18.16b, v16.16b // -m neg v19.16b, v17.16b ld1 {v0.8h, v1.8h}, [x0] sxtl v16.8h, v18.8b sxtl2 v17.8h, v18.16b sxtl v18.8h, v19.8b sxtl2 v19.8h, v19.16b ld1 {v2.8h, v3.8h}, [x8] shl v16.8h, v16.8h, #9 // -m << 9 shl v17.8h, v17.8h, #9 shl v18.8h, v18.8h, #9 shl v19.8h, v19.8h, #9 sub v4.8h, v0.8h, v4.8h // a - b sub v5.8h, v1.8h, v5.8h sub v6.8h, v2.8h, v6.8h sub v7.8h, v3.8h, v7.8h sqrdmulh v4.8h, v4.8h, v16.8h // ((a-b)*-m + 32) >> 6 sqrdmulh v5.8h, v5.8h, v17.8h sqrdmulh v6.8h, v6.8h, v18.8h sqrdmulh v7.8h, v7.8h, v19.8h add v0.8h, v0.8h, v4.8h add v1.8h, v1.8h, v5.8h add v2.8h, v2.8h, v6.8h add v3.8h, v3.8h, v7.8h st1 {v0.8h, v1.8h}, [x0], x1 st1 {v2.8h, v3.8h}, [x8], x1 b.gt 16b ret 32: AARCH64_VALID_JUMP_TARGET ld1 {v16.16b, v17.16b}, [x5], #32 ld1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x2], #64 subs w4, w4, #1 neg v18.16b, v16.16b // -m neg v19.16b, v17.16b sxtl v16.8h, v18.8b sxtl2 v17.8h, v18.16b sxtl v18.8h, v19.8b sxtl2 v19.8h, v19.16b ld1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0] shl v16.8h, v16.8h, #9 // -m << 9 shl v17.8h, v17.8h, #9 shl v18.8h, v18.8h, #9 shl v19.8h, v19.8h, #9 sub v4.8h, v0.8h, v4.8h // a - b sub v5.8h, v1.8h, v5.8h sub v6.8h, v2.8h, v6.8h sub v7.8h, v3.8h, v7.8h sqrdmulh v4.8h, v4.8h, v16.8h // ((a-b)*-m + 32) >> 6 sqrdmulh v5.8h, v5.8h, v17.8h sqrdmulh v6.8h, v6.8h, v18.8h sqrdmulh v7.8h, v7.8h, v19.8h add v0.8h, v0.8h, v4.8h add v1.8h, v1.8h, v5.8h add v2.8h, v2.8h, v6.8h add v3.8h, v3.8h, v7.8h st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], x1 b.gt 32b ret L(blend_tbl): .hword L(blend_tbl) - 32b .hword L(blend_tbl) - 160b .hword L(blend_tbl) - 80b .hword L(blend_tbl) - 40b endfunc function blend_h_16bpc_neon, export=1 adr x6, L(blend_h_tbl) movrel x5, X(obmc_masks) add x5, x5, w4, uxtw sub w4, w4, w4, lsr #2 clz w7, w3 add x8, x0, x1 lsl x1, x1, #1 sub w7, w7, #24 ldrh w7, [x6, x7, lsl #1] sub x6, x6, w7, uxtw br x6 2: AARCH64_VALID_JUMP_TARGET ld2r {v2.8b, v3.8b}, [x5], #2 ld1 {v1.4h}, [x2], #8 ext v2.8b, v2.8b, v3.8b, #6 subs w4, w4, #2 neg v2.8b, v2.8b // -m ld1 {v0.s}[0], [x0] ld1 {v0.s}[1], [x8] sxtl v2.8h, v2.8b shl v2.4h, v2.4h, #9 // -m << 9 sub v1.4h, v0.4h, v1.4h // a - b sqrdmulh v1.4h, v1.4h, v2.4h // ((a-b)*-m + 32) >> 6 add v0.4h, v0.4h, v1.4h st1 {v0.s}[0], [x0], x1 st1 {v0.s}[1], [x8], x1 b.gt 2b ret 4: AARCH64_VALID_JUMP_TARGET ld2r {v2.8b, v3.8b}, [x5], #2 ld1 {v1.8h}, [x2], #16 ext v2.8b, v2.8b, v3.8b, #4 subs w4, w4, #2 neg v2.8b, v2.8b // -m ld1 {v0.d}[0], [x0] ld1 {v0.d}[1], [x8] sxtl v2.8h, v2.8b shl v2.8h, v2.8h, #9 // -m << 9 sub v1.8h, v0.8h, v1.8h // a - b sqrdmulh v1.8h, v1.8h, v2.8h // ((a-b)*-m + 32) >> 6 add v0.8h, v0.8h, v1.8h st1 {v0.d}[0], [x0], x1 st1 {v0.d}[1], [x8], x1 b.gt 4b ret 8: AARCH64_VALID_JUMP_TARGET ld2r {v4.8b, v5.8b}, [x5], #2 ld1 {v2.8h, v3.8h}, [x2], #32 neg v4.8b, v4.8b // -m neg v5.8b, v5.8b ld1 {v0.8h}, [x0] subs w4, w4, #2 sxtl v4.8h, v4.8b sxtl v5.8h, v5.8b ld1 {v1.8h}, [x8] shl v4.8h, v4.8h, #9 // -m << 9 shl v5.8h, v5.8h, #9 sub v2.8h, v0.8h, v2.8h // a - b sub v3.8h, v1.8h, v3.8h sqrdmulh v2.8h, v2.8h, v4.8h // ((a-b)*-m + 32) >> 6 sqrdmulh v3.8h, v3.8h, v5.8h add v0.8h, v0.8h, v2.8h add v1.8h, v1.8h, v3.8h st1 {v0.8h}, [x0], x1 st1 {v1.8h}, [x8], x1 b.gt 8b ret 16: AARCH64_VALID_JUMP_TARGET ld2r {v16.8b, v17.8b}, [x5], #2 ld1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x2], #64 neg v16.8b, v16.8b // -m neg v17.8b, v17.8b ld1 {v0.8h, v1.8h}, [x0] ld1 {v2.8h, v3.8h}, [x8] subs w4, w4, #2 sxtl v16.8h, v16.8b sxtl v17.8h, v17.8b shl v16.8h, v16.8h, #9 // -m << 9 shl v17.8h, v17.8h, #9 sub v4.8h, v0.8h, v4.8h // a - b sub v5.8h, v1.8h, v5.8h sub v6.8h, v2.8h, v6.8h sub v7.8h, v3.8h, v7.8h sqrdmulh v4.8h, v4.8h, v16.8h // ((a-b)*-m + 32) >> 6 sqrdmulh v5.8h, v5.8h, v16.8h sqrdmulh v6.8h, v6.8h, v17.8h sqrdmulh v7.8h, v7.8h, v17.8h add v0.8h, v0.8h, v4.8h add v1.8h, v1.8h, v5.8h add v2.8h, v2.8h, v6.8h add v3.8h, v3.8h, v7.8h st1 {v0.8h, v1.8h}, [x0], x1 st1 {v2.8h, v3.8h}, [x8], x1 b.gt 16b ret 1280: 640: 320: AARCH64_VALID_JUMP_TARGET sub x1, x1, w3, uxtw #1 add x7, x2, w3, uxtw #1 321: ld2r {v24.8b, v25.8b}, [x5], #2 mov w6, w3 neg v24.8b, v24.8b // -m neg v25.8b, v25.8b sxtl v24.8h, v24.8b sxtl v25.8h, v25.8b shl v24.8h, v24.8h, #9 // -m << 9 shl v25.8h, v25.8h, #9 32: ld1 {v16.8h, v17.8h, v18.8h, v19.8h}, [x2], #64 ld1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0] subs w6, w6, #32 sub v16.8h, v0.8h, v16.8h // a - b sub v17.8h, v1.8h, v17.8h sub v18.8h, v2.8h, v18.8h sub v19.8h, v3.8h, v19.8h ld1 {v20.8h, v21.8h, v22.8h, v23.8h}, [x7], #64 ld1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x8] sqrdmulh v16.8h, v16.8h, v24.8h // ((a-b)*-m + 32) >> 6 sqrdmulh v17.8h, v17.8h, v24.8h sqrdmulh v18.8h, v18.8h, v24.8h sqrdmulh v19.8h, v19.8h, v24.8h sub v20.8h, v4.8h, v20.8h // a - b sub v21.8h, v5.8h, v21.8h sub v22.8h, v6.8h, v22.8h sub v23.8h, v7.8h, v23.8h add v0.8h, v0.8h, v16.8h add v1.8h, v1.8h, v17.8h add v2.8h, v2.8h, v18.8h add v3.8h, v3.8h, v19.8h sqrdmulh v20.8h, v20.8h, v25.8h // ((a-b)*-m + 32) >> 6 sqrdmulh v21.8h, v21.8h, v25.8h sqrdmulh v22.8h, v22.8h, v25.8h sqrdmulh v23.8h, v23.8h, v25.8h st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 add v4.8h, v4.8h, v20.8h add v5.8h, v5.8h, v21.8h add v6.8h, v6.8h, v22.8h add v7.8h, v7.8h, v23.8h st1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x8], #64 b.gt 32b subs w4, w4, #2 add x0, x0, x1 add x8, x8, x1 add x2, x2, w3, uxtw #1 add x7, x7, w3, uxtw #1 b.gt 321b ret L(blend_h_tbl): .hword L(blend_h_tbl) - 1280b .hword L(blend_h_tbl) - 640b .hword L(blend_h_tbl) - 320b .hword L(blend_h_tbl) - 16b .hword L(blend_h_tbl) - 8b .hword L(blend_h_tbl) - 4b .hword L(blend_h_tbl) - 2b endfunc function blend_v_16bpc_neon, export=1 adr x6, L(blend_v_tbl) movrel x5, X(obmc_masks) add x5, x5, w3, uxtw clz w3, w3 add x8, x0, x1 lsl x1, x1, #1 sub w3, w3, #26 ldrh w3, [x6, x3, lsl #1] sub x6, x6, w3, uxtw br x6 20: AARCH64_VALID_JUMP_TARGET ld1r {v2.8b}, [x5] neg v2.8b, v2.8b // -m sxtl v2.8h, v2.8b shl v2.4h, v2.4h, #9 // -m << 9 2: ld1 {v1.s}[0], [x2], #4 ld1 {v0.h}[0], [x0] subs w4, w4, #2 ld1 {v1.h}[1], [x2] ld1 {v0.h}[1], [x8] add x2, x2, #4 sub v1.4h, v0.4h, v1.4h // a - b sqrdmulh v1.4h, v1.4h, v2.4h // ((a-b)*-m + 32) >> 6 add v0.4h, v0.4h, v1.4h st1 {v0.h}[0], [x0], x1 st1 {v0.h}[1], [x8], x1 b.gt 2b ret 40: AARCH64_VALID_JUMP_TARGET ld1r {v2.2s}, [x5] sub x1, x1, #4 neg v2.8b, v2.8b // -m sxtl v2.8h, v2.8b shl v2.8h, v2.8h, #9 // -m << 9 4: ld1 {v1.8h}, [x2], #16 ld1 {v0.d}[0], [x0] ld1 {v0.d}[1], [x8] subs w4, w4, #2 sub v1.8h, v0.8h, v1.8h // a - b sqrdmulh v1.8h, v1.8h, v2.8h // ((a-b)*-m + 32) >> 6 add v0.8h, v0.8h, v1.8h st1 {v0.s}[0], [x0], #4 st1 {v0.s}[2], [x8], #4 st1 {v0.h}[2], [x0], x1 st1 {v0.h}[6], [x8], x1 b.gt 4b ret 80: AARCH64_VALID_JUMP_TARGET ld1 {v4.8b}, [x5] sub x1, x1, #8 neg v4.8b, v4.8b // -m sxtl v4.8h, v4.8b shl v4.8h, v4.8h, #9 // -m << 9 8: ld1 {v2.8h, v3.8h}, [x2], #32 ld1 {v0.8h}, [x0] ld1 {v1.8h}, [x8] subs w4, w4, #2 sub v2.8h, v0.8h, v2.8h // a - b sub v3.8h, v1.8h, v3.8h sqrdmulh v2.8h, v2.8h, v4.8h // ((a-b)*-m + 32) >> 6 sqrdmulh v3.8h, v3.8h, v4.8h add v0.8h, v0.8h, v2.8h add v1.8h, v1.8h, v3.8h st1 {v0.d}[0], [x0], #8 st1 {v1.d}[0], [x8], #8 st1 {v0.s}[2], [x0], x1 st1 {v1.s}[2], [x8], x1 b.gt 8b ret 160: AARCH64_VALID_JUMP_TARGET ld1 {v16.16b}, [x5] sub x1, x1, #16 neg v17.16b, v16.16b // -m sxtl v16.8h, v17.8b sxtl2 v17.8h, v17.16b shl v16.8h, v16.8h, #9 // -m << 9 shl v17.4h, v17.4h, #9 16: ld1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x2], #64 ld1 {v0.8h, v1.8h}, [x0] subs w4, w4, #2 ld1 {v2.8h, v3.8h}, [x8] sub v4.8h, v0.8h, v4.8h // a - b sub v5.4h, v1.4h, v5.4h sub v6.8h, v2.8h, v6.8h sub v7.4h, v3.4h, v7.4h sqrdmulh v4.8h, v4.8h, v16.8h // ((a-b)*-m + 32) >> 6 sqrdmulh v5.4h, v5.4h, v17.4h sqrdmulh v6.8h, v6.8h, v16.8h sqrdmulh v7.4h, v7.4h, v17.4h add v0.8h, v0.8h, v4.8h add v1.4h, v1.4h, v5.4h add v2.8h, v2.8h, v6.8h add v3.4h, v3.4h, v7.4h st1 {v0.8h}, [x0], #16 st1 {v2.8h}, [x8], #16 st1 {v1.4h}, [x0], x1 st1 {v3.4h}, [x8], x1 b.gt 16b ret 320: AARCH64_VALID_JUMP_TARGET ld1 {v24.16b, v25.16b}, [x5] neg v26.16b, v24.16b // -m neg v27.8b, v25.8b sxtl v24.8h, v26.8b sxtl2 v25.8h, v26.16b sxtl v26.8h, v27.8b shl v24.8h, v24.8h, #9 // -m << 9 shl v25.8h, v25.8h, #9 shl v26.8h, v26.8h, #9 32: ld1 {v16.8h, v17.8h, v18.8h, v19.8h}, [x2], #64 ld1 {v0.8h, v1.8h, v2.8h}, [x0] ld1 {v20.8h, v21.8h, v22.8h, v23.8h}, [x2], #64 ld1 {v4.8h, v5.8h, v6.8h}, [x8] subs w4, w4, #2 sub v16.8h, v0.8h, v16.8h // a - b sub v17.8h, v1.8h, v17.8h sub v18.8h, v2.8h, v18.8h sub v20.8h, v4.8h, v20.8h sub v21.8h, v5.8h, v21.8h sub v22.8h, v6.8h, v22.8h sqrdmulh v16.8h, v16.8h, v24.8h // ((a-b)*-m + 32) >> 6 sqrdmulh v17.8h, v17.8h, v25.8h sqrdmulh v18.8h, v18.8h, v26.8h sqrdmulh v20.8h, v20.8h, v24.8h sqrdmulh v21.8h, v21.8h, v25.8h sqrdmulh v22.8h, v22.8h, v26.8h add v0.8h, v0.8h, v16.8h add v1.8h, v1.8h, v17.8h add v2.8h, v2.8h, v18.8h add v4.8h, v4.8h, v20.8h add v5.8h, v5.8h, v21.8h add v6.8h, v6.8h, v22.8h st1 {v0.8h, v1.8h, v2.8h}, [x0], x1 st1 {v4.8h, v5.8h, v6.8h}, [x8], x1 b.gt 32b ret L(blend_v_tbl): .hword L(blend_v_tbl) - 320b .hword L(blend_v_tbl) - 160b .hword L(blend_v_tbl) - 80b .hword L(blend_v_tbl) - 40b .hword L(blend_v_tbl) - 20b endfunc // This has got the same signature as the put_8tap functions, // and assumes that x9 is set to (clz(w)-24). function put_neon adr x10, L(put_tbl) ldrh w9, [x10, x9, lsl #1] sub x10, x10, w9, uxtw br x10 2: AARCH64_VALID_JUMP_TARGET ld1 {v0.s}[0], [x2], x3 ld1 {v1.s}[0], [x2], x3 subs w5, w5, #2 st1 {v0.s}[0], [x0], x1 st1 {v1.s}[0], [x0], x1 b.gt 2b ret 4: AARCH64_VALID_JUMP_TARGET ld1 {v0.4h}, [x2], x3 ld1 {v1.4h}, [x2], x3 subs w5, w5, #2 st1 {v0.4h}, [x0], x1 st1 {v1.4h}, [x0], x1 b.gt 4b ret 80: AARCH64_VALID_JUMP_TARGET add x8, x0, x1 lsl x1, x1, #1 add x9, x2, x3 lsl x3, x3, #1 8: ld1 {v0.8h}, [x2], x3 ld1 {v1.8h}, [x9], x3 subs w5, w5, #2 st1 {v0.8h}, [x0], x1 st1 {v1.8h}, [x8], x1 b.gt 8b ret 16: AARCH64_VALID_JUMP_TARGET ldp x6, x7, [x2] ldp x8, x9, [x2, #16] stp x6, x7, [x0] subs w5, w5, #1 stp x8, x9, [x0, #16] add x2, x2, x3 add x0, x0, x1 b.gt 16b ret 32: AARCH64_VALID_JUMP_TARGET ldp x6, x7, [x2] ldp x8, x9, [x2, #16] stp x6, x7, [x0] ldp x10, x11, [x2, #32] stp x8, x9, [x0, #16] subs w5, w5, #1 ldp x12, x13, [x2, #48] stp x10, x11, [x0, #32] stp x12, x13, [x0, #48] add x2, x2, x3 add x0, x0, x1 b.gt 32b ret 64: AARCH64_VALID_JUMP_TARGET ldp q0, q1, [x2] ldp q2, q3, [x2, #32] stp q0, q1, [x0] ldp q4, q5, [x2, #64] stp q2, q3, [x0, #32] ldp q6, q7, [x2, #96] subs w5, w5, #1 stp q4, q5, [x0, #64] stp q6, q7, [x0, #96] add x2, x2, x3 add x0, x0, x1 b.gt 64b ret 128: AARCH64_VALID_JUMP_TARGET ldp q0, q1, [x2] ldp q2, q3, [x2, #32] stp q0, q1, [x0] ldp q4, q5, [x2, #64] stp q2, q3, [x0, #32] ldp q6, q7, [x2, #96] subs w5, w5, #1 stp q4, q5, [x0, #64] ldp q16, q17, [x2, #128] stp q6, q7, [x0, #96] ldp q18, q19, [x2, #160] stp q16, q17, [x0, #128] ldp q20, q21, [x2, #192] stp q18, q19, [x0, #160] ldp q22, q23, [x2, #224] stp q20, q21, [x0, #192] stp q22, q23, [x0, #224] add x2, x2, x3 add x0, x0, x1 b.gt 128b ret L(put_tbl): .hword L(put_tbl) - 128b .hword L(put_tbl) - 64b .hword L(put_tbl) - 32b .hword L(put_tbl) - 16b .hword L(put_tbl) - 80b .hword L(put_tbl) - 4b .hword L(put_tbl) - 2b endfunc // This has got the same signature as the prep_8tap functions, // and assumes that x9 is set to (clz(w)-24), w7 to intermediate_bits and // x8 to w*2. function prep_neon adr x10, L(prep_tbl) ldrh w9, [x10, x9, lsl #1] dup v31.8h, w7 // intermediate_bits movi v30.8h, #(PREP_BIAS >> 8), lsl #8 sub x10, x10, w9, uxtw br x10 40: AARCH64_VALID_JUMP_TARGET add x9, x1, x2 lsl x2, x2, #1 4: ld1 {v0.d}[0], [x1], x2 ld1 {v0.d}[1], [x9], x2 subs w4, w4, #2 sshl v0.8h, v0.8h, v31.8h sub v0.8h, v0.8h, v30.8h st1 {v0.8h}, [x0], #16 b.gt 4b ret 80: AARCH64_VALID_JUMP_TARGET add x9, x1, x2 lsl x2, x2, #1 8: ld1 {v0.8h}, [x1], x2 ld1 {v1.8h}, [x9], x2 subs w4, w4, #2 sshl v0.8h, v0.8h, v31.8h sshl v1.8h, v1.8h, v31.8h sub v0.8h, v0.8h, v30.8h sub v1.8h, v1.8h, v30.8h st1 {v0.8h, v1.8h}, [x0], #32 b.gt 8b ret 16: AARCH64_VALID_JUMP_TARGET ldp q0, q1, [x1] add x1, x1, x2 sshl v0.8h, v0.8h, v31.8h ldp q2, q3, [x1] add x1, x1, x2 subs w4, w4, #2 sshl v1.8h, v1.8h, v31.8h sshl v2.8h, v2.8h, v31.8h sshl v3.8h, v3.8h, v31.8h sub v0.8h, v0.8h, v30.8h sub v1.8h, v1.8h, v30.8h sub v2.8h, v2.8h, v30.8h sub v3.8h, v3.8h, v30.8h st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 b.gt 16b ret 32: AARCH64_VALID_JUMP_TARGET ldp q0, q1, [x1] sshl v0.8h, v0.8h, v31.8h ldp q2, q3, [x1, #32] add x1, x1, x2 sshl v1.8h, v1.8h, v31.8h sshl v2.8h, v2.8h, v31.8h sshl v3.8h, v3.8h, v31.8h subs w4, w4, #1 sub v0.8h, v0.8h, v30.8h sub v1.8h, v1.8h, v30.8h sub v2.8h, v2.8h, v30.8h sub v3.8h, v3.8h, v30.8h st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 b.gt 32b ret 64: AARCH64_VALID_JUMP_TARGET ldp q0, q1, [x1] subs w4, w4, #1 sshl v0.8h, v0.8h, v31.8h ldp q2, q3, [x1, #32] sshl v1.8h, v1.8h, v31.8h ldp q4, q5, [x1, #64] sshl v2.8h, v2.8h, v31.8h sshl v3.8h, v3.8h, v31.8h ldp q6, q7, [x1, #96] add x1, x1, x2 sshl v4.8h, v4.8h, v31.8h sshl v5.8h, v5.8h, v31.8h sshl v6.8h, v6.8h, v31.8h sshl v7.8h, v7.8h, v31.8h sub v0.8h, v0.8h, v30.8h sub v1.8h, v1.8h, v30.8h sub v2.8h, v2.8h, v30.8h sub v3.8h, v3.8h, v30.8h stp q0, q1, [x0] sub v4.8h, v4.8h, v30.8h sub v5.8h, v5.8h, v30.8h stp q2, q3, [x0, #32] sub v6.8h, v6.8h, v30.8h sub v7.8h, v7.8h, v30.8h stp q4, q5, [x0, #64] stp q6, q7, [x0, #96] add x0, x0, x8 b.gt 64b ret 128: AARCH64_VALID_JUMP_TARGET ldp q0, q1, [x1] subs w4, w4, #1 sshl v0.8h, v0.8h, v31.8h ldp q2, q3, [x1, #32] sshl v1.8h, v1.8h, v31.8h ldp q4, q5, [x1, #64] sshl v2.8h, v2.8h, v31.8h sshl v3.8h, v3.8h, v31.8h ldp q6, q7, [x1, #96] sshl v4.8h, v4.8h, v31.8h sshl v5.8h, v5.8h, v31.8h ldp q16, q17, [x1, #128] sshl v6.8h, v6.8h, v31.8h sshl v7.8h, v7.8h, v31.8h ldp q18, q19, [x1, #160] sshl v16.8h, v16.8h, v31.8h sshl v17.8h, v17.8h, v31.8h ldp q20, q21, [x1, #192] sshl v18.8h, v18.8h, v31.8h sshl v19.8h, v19.8h, v31.8h ldp q22, q23, [x1, #224] add x1, x1, x2 sshl v20.8h, v20.8h, v31.8h sshl v21.8h, v21.8h, v31.8h sshl v22.8h, v22.8h, v31.8h sshl v23.8h, v23.8h, v31.8h sub v0.8h, v0.8h, v30.8h sub v1.8h, v1.8h, v30.8h sub v2.8h, v2.8h, v30.8h sub v3.8h, v3.8h, v30.8h stp q0, q1, [x0] sub v4.8h, v4.8h, v30.8h sub v5.8h, v5.8h, v30.8h stp q2, q3, [x0, #32] sub v6.8h, v6.8h, v30.8h sub v7.8h, v7.8h, v30.8h stp q4, q5, [x0, #64] sub v16.8h, v16.8h, v30.8h sub v17.8h, v17.8h, v30.8h stp q6, q7, [x0, #96] sub v18.8h, v18.8h, v30.8h sub v19.8h, v19.8h, v30.8h stp q16, q17, [x0, #128] sub v20.8h, v20.8h, v30.8h sub v21.8h, v21.8h, v30.8h stp q18, q19, [x0, #160] sub v22.8h, v22.8h, v30.8h sub v23.8h, v23.8h, v30.8h stp q20, q21, [x0, #192] stp q22, q23, [x0, #224] add x0, x0, x8 b.gt 128b ret L(prep_tbl): .hword L(prep_tbl) - 128b .hword L(prep_tbl) - 64b .hword L(prep_tbl) - 32b .hword L(prep_tbl) - 16b .hword L(prep_tbl) - 80b .hword L(prep_tbl) - 40b endfunc .macro load_slice s0, s1, strd, wd, d0, d1, d2, d3, d4, d5, d6 ld1 {\d0\wd}[0], [\s0], \strd ld1 {\d1\wd}[0], [\s1], \strd .ifnb \d2 ld1 {\d2\wd}[0], [\s0], \strd ld1 {\d3\wd}[0], [\s1], \strd .endif .ifnb \d4 ld1 {\d4\wd}[0], [\s0], \strd .endif .ifnb \d5 ld1 {\d5\wd}[0], [\s1], \strd .endif .ifnb \d6 ld1 {\d6\wd}[0], [\s0], \strd .endif .endm .macro load_reg s0, s1, strd, wd, d0, d1, d2, d3, d4, d5, d6 ld1 {\d0\wd}, [\s0], \strd ld1 {\d1\wd}, [\s1], \strd .ifnb \d2 ld1 {\d2\wd}, [\s0], \strd ld1 {\d3\wd}, [\s1], \strd .endif .ifnb \d4 ld1 {\d4\wd}, [\s0], \strd .endif .ifnb \d5 ld1 {\d5\wd}, [\s1], \strd .endif .ifnb \d6 ld1 {\d6\wd}, [\s0], \strd .endif .endm .macro load_regpair s0, s1, strd, wd, d0, d1, d2, d3, d4, d5 ld1 {\d0\wd, \d1\wd}, [\s0], \strd .ifnb \d2 ld1 {\d2\wd, \d3\wd}, [\s1], \strd .endif .ifnb \d4 ld1 {\d4\wd, \d5\wd}, [\s0], \strd .endif .endm .macro load_s s0, s1, strd, d0, d1, d2, d3, d4, d5, d6 load_slice \s0, \s1, \strd, .s, \d0, \d1, \d2, \d3, \d4, \d5, \d6 .endm .macro load_4h s0, s1, strd, d0, d1, d2, d3, d4, d5, d6 load_reg \s0, \s1, \strd, .4h, \d0, \d1, \d2, \d3, \d4, \d5, \d6 .endm .macro load_8h s0, s1, strd, d0, d1, d2, d3, d4, d5, d6 load_reg \s0, \s1, \strd, .8h, \d0, \d1, \d2, \d3, \d4, \d5, \d6 .endm .macro load_16h s0, s1, strd, d0, d1, d2, d3, d4, d5 load_regpair \s0, \s1, \strd, .8h, \d0, \d1, \d2, \d3, \d4, \d5 .endm .macro interleave_1 wd, r0, r1, r2, r3, r4 trn1 \r0\wd, \r0\wd, \r1\wd trn1 \r1\wd, \r1\wd, \r2\wd .ifnb \r3 trn1 \r2\wd, \r2\wd, \r3\wd trn1 \r3\wd, \r3\wd, \r4\wd .endif .endm .macro interleave_1_s r0, r1, r2, r3, r4 interleave_1 .2s, \r0, \r1, \r2, \r3, \r4 .endm .macro umin_h c, wd, r0, r1, r2, r3 umin \r0\wd, \r0\wd, \c\wd .ifnb \r1 umin \r1\wd, \r1\wd, \c\wd .endif .ifnb \r2 umin \r2\wd, \r2\wd, \c\wd umin \r3\wd, \r3\wd, \c\wd .endif .endm .macro sub_h c, wd, r0, r1, r2, r3 sub \r0\wd, \r0\wd, \c\wd .ifnb \r1 sub \r1\wd, \r1\wd, \c\wd .endif .ifnb \r2 sub \r2\wd, \r2\wd, \c\wd sub \r3\wd, \r3\wd, \c\wd .endif .endm .macro smull_smlal_4 d, s0, s1, s2, s3 smull \d\().4s, \s0\().4h, v0.h[0] smlal \d\().4s, \s1\().4h, v0.h[1] smlal \d\().4s, \s2\().4h, v0.h[2] smlal \d\().4s, \s3\().4h, v0.h[3] .endm .macro smull2_smlal2_4 d, s0, s1, s2, s3 smull2 \d\().4s, \s0\().8h, v0.h[0] smlal2 \d\().4s, \s1\().8h, v0.h[1] smlal2 \d\().4s, \s2\().8h, v0.h[2] smlal2 \d\().4s, \s3\().8h, v0.h[3] .endm .macro smull_smlal_8 d, s0, s1, s2, s3, s4, s5, s6, s7 smull \d\().4s, \s0\().4h, v0.h[0] smlal \d\().4s, \s1\().4h, v0.h[1] smlal \d\().4s, \s2\().4h, v0.h[2] smlal \d\().4s, \s3\().4h, v0.h[3] smlal \d\().4s, \s4\().4h, v0.h[4] smlal \d\().4s, \s5\().4h, v0.h[5] smlal \d\().4s, \s6\().4h, v0.h[6] smlal \d\().4s, \s7\().4h, v0.h[7] .endm .macro smull2_smlal2_8 d, s0, s1, s2, s3, s4, s5, s6, s7 smull2 \d\().4s, \s0\().8h, v0.h[0] smlal2 \d\().4s, \s1\().8h, v0.h[1] smlal2 \d\().4s, \s2\().8h, v0.h[2] smlal2 \d\().4s, \s3\().8h, v0.h[3] smlal2 \d\().4s, \s4\().8h, v0.h[4] smlal2 \d\().4s, \s5\().8h, v0.h[5] smlal2 \d\().4s, \s6\().8h, v0.h[6] smlal2 \d\().4s, \s7\().8h, v0.h[7] .endm .macro sqrshrun_h shift, r0, r1, r2, r3 sqrshrun \r0\().4h, \r0\().4s, #\shift .ifnb \r1 sqrshrun2 \r0\().8h, \r1\().4s, #\shift .endif .ifnb \r2 sqrshrun \r2\().4h, \r2\().4s, #\shift sqrshrun2 \r2\().8h, \r3\().4s, #\shift .endif .endm .macro xtn_h r0, r1, r2, r3 uzp1 \r0\().8h, \r0\().8h, \r1\().8h // Same as xtn, xtn2 .ifnb \r2 uzp1 \r2\().8h, \r2\().8h, \r3\().8h // Ditto .endif .endm .macro srshl_s shift, r0, r1, r2, r3 srshl \r0\().4s, \r0\().4s, \shift\().4s srshl \r1\().4s, \r1\().4s, \shift\().4s .ifnb \r2 srshl \r2\().4s, \r2\().4s, \shift\().4s srshl \r3\().4s, \r3\().4s, \shift\().4s .endif .endm .macro st_s strd, reg, lanes st1 {\reg\().s}[0], [x0], \strd st1 {\reg\().s}[1], [x9], \strd .if \lanes > 2 st1 {\reg\().s}[2], [x0], \strd st1 {\reg\().s}[3], [x9], \strd .endif .endm .macro st_d strd, r0, r1 st1 {\r0\().d}[0], [x0], \strd st1 {\r0\().d}[1], [x9], \strd .ifnb \r1 st1 {\r1\().d}[0], [x0], \strd st1 {\r1\().d}[1], [x9], \strd .endif .endm .macro shift_store_4 type, strd, r0, r1, r2, r3 .ifc \type, put sqrshrun_h 6, \r0, \r1, \r2, \r3 umin_h v31, .8h, \r0, \r2 .else srshl_s v30, \r0, \r1, \r2, \r3 // -(6-intermediate_bits) xtn_h \r0, \r1, \r2, \r3 sub_h v29, .8h, \r0, \r2 // PREP_BIAS .endif st_d \strd, \r0, \r2 .endm .macro st_reg strd, wd, r0, r1, r2, r3, r4, r5, r6, r7 st1 {\r0\wd}, [x0], \strd st1 {\r1\wd}, [x9], \strd .ifnb \r2 st1 {\r2\wd}, [x0], \strd st1 {\r3\wd}, [x9], \strd .endif .ifnb \r4 st1 {\r4\wd}, [x0], \strd st1 {\r5\wd}, [x9], \strd st1 {\r6\wd}, [x0], \strd st1 {\r7\wd}, [x9], \strd .endif .endm .macro st_8h strd, r0, r1, r2, r3, r4, r5, r6, r7 st_reg \strd, .8h, \r0, \r1, \r2, \r3, \r4, \r5, \r6, \r7 .endm .macro shift_store_8 type, strd, r0, r1, r2, r3 .ifc \type, put sqrshrun_h 6, \r0, \r1, \r2, \r3 umin_h v31, .8h, \r0, \r2 .else srshl_s v30, \r0, \r1, \r2, \r3 // -(6-intermediate_bits) xtn_h \r0, \r1, \r2, \r3 sub_h v29, .8h, \r0, \r2 // PREP_BIAS .endif st_8h \strd, \r0, \r2 .endm .macro shift_store_16 type, strd, dst, r0, r1, r2, r3 .ifc \type, put sqrshrun_h 6, \r0, \r1, \r2, \r3 umin \r0\().8h, \r0\().8h, v31.8h umin \r1\().8h, \r2\().8h, v31.8h .else srshl_s v30, \r0, \r1, \r2, \r3 // -(6-intermediate_bits) xtn_h \r0, \r1, \r2, \r3 sub \r0\().8h, \r0\().8h, v29.8h sub \r1\().8h, \r2\().8h, v29.8h .endif st1 {\r0\().8h, \r1\().8h}, [\dst], \strd .endm .macro make_8tap_fn op, type, type_h, type_v function \op\()_8tap_\type\()_16bpc_neon, export=1 mov w9, \type_h mov w10, \type_v b \op\()_8tap_neon endfunc .endm // No spaces in these expressions, due to gas-preprocessor. #define REGULAR ((0*15<<7)|3*15) #define SMOOTH ((1*15<<7)|4*15) #define SHARP ((2*15<<7)|3*15) .macro filter_fn type, dst, d_strd, src, s_strd, w, h, mx, xmx, my, xmy, bdmax, ds2, sr2 make_8tap_fn \type, regular, REGULAR, REGULAR make_8tap_fn \type, regular_smooth, REGULAR, SMOOTH make_8tap_fn \type, regular_sharp, REGULAR, SHARP make_8tap_fn \type, smooth, SMOOTH, SMOOTH make_8tap_fn \type, smooth_regular, SMOOTH, REGULAR make_8tap_fn \type, smooth_sharp, SMOOTH, SHARP make_8tap_fn \type, sharp, SHARP, SHARP make_8tap_fn \type, sharp_regular, SHARP, REGULAR make_8tap_fn \type, sharp_smooth, SHARP, SMOOTH function \type\()_8tap_neon .ifc \bdmax, w8 ldr w8, [sp] .endif mov w11, #0x4081 // (1 << 14) | (1 << 7) | (1 << 0) mul \mx, \mx, w11 mul \my, \my, w11 add \mx, \mx, w9 // mx, 8tap_h, 4tap_h add \my, \my, w10 // my, 8tap_v, 4tap_v .ifc \type, prep uxtw \d_strd, \w lsl \d_strd, \d_strd, #1 .endif dup v31.8h, \bdmax // bitdepth_max clz \bdmax, \bdmax clz w9, \w sub \bdmax, \bdmax, #18 // intermediate_bits = clz(bitdepth_max) - 18 mov w12, #6 tst \mx, #(0x7f << 14) sub w9, w9, #24 add w13, w12, \bdmax // 6 + intermediate_bits sub w12, w12, \bdmax // 6 - intermediate_bits movrel x11, X(mc_subpel_filters), -8 b.ne L(\type\()_8tap_h) tst \my, #(0x7f << 14) b.ne L(\type\()_8tap_v) b \type\()_neon L(\type\()_8tap_h): cmp \w, #4 ubfx w10, \mx, #7, #7 and \mx, \mx, #0x7f b.le 4f mov \mx, w10 4: tst \my, #(0x7f << 14) add \xmx, x11, \mx, uxtw #3 b.ne L(\type\()_8tap_hv) adr x10, L(\type\()_8tap_h_tbl) dup v30.4s, w12 // 6 - intermediate_bits ldrh w9, [x10, x9, lsl #1] neg v30.4s, v30.4s // -(6-intermediate_bits) .ifc \type, put dup v29.8h, \bdmax // intermediate_bits .else movi v28.8h, #(PREP_BIAS >> 8), lsl #8 .endif sub x10, x10, w9, uxtw .ifc \type, put neg v29.8h, v29.8h // -intermediate_bits .endif br x10 20: // 2xN h AARCH64_VALID_JUMP_TARGET .ifc \type, put add \xmx, \xmx, #2 ld1 {v0.s}[0], [\xmx] sub \src, \src, #2 add \ds2, \dst, \d_strd add \sr2, \src, \s_strd lsl \d_strd, \d_strd, #1 lsl \s_strd, \s_strd, #1 sxtl v0.8h, v0.8b 2: ld1 {v4.8h}, [\src], \s_strd ld1 {v6.8h}, [\sr2], \s_strd ext v5.16b, v4.16b, v4.16b, #2 ext v7.16b, v6.16b, v6.16b, #2 subs \h, \h, #2 trn1 v3.2s, v4.2s, v6.2s trn2 v6.2s, v4.2s, v6.2s trn1 v4.2s, v5.2s, v7.2s trn2 v7.2s, v5.2s, v7.2s smull v3.4s, v3.4h, v0.h[0] smlal v3.4s, v4.4h, v0.h[1] smlal v3.4s, v6.4h, v0.h[2] smlal v3.4s, v7.4h, v0.h[3] srshl v3.4s, v3.4s, v30.4s // -(6-intermediate_bits) sqxtun v3.4h, v3.4s srshl v3.4h, v3.4h, v29.4h // -intermediate_bits umin v3.4h, v3.4h, v31.4h st1 {v3.s}[0], [\dst], \d_strd st1 {v3.s}[1], [\ds2], \d_strd b.gt 2b ret .endif 40: // 4xN h AARCH64_VALID_JUMP_TARGET add \xmx, \xmx, #2 ld1 {v0.s}[0], [\xmx] sub \src, \src, #2 add \ds2, \dst, \d_strd add \sr2, \src, \s_strd lsl \d_strd, \d_strd, #1 lsl \s_strd, \s_strd, #1 sxtl v0.8h, v0.8b 4: ld1 {v16.8h}, [\src], \s_strd ld1 {v20.8h}, [\sr2], \s_strd ext v17.16b, v16.16b, v16.16b, #2 ext v18.16b, v16.16b, v16.16b, #4 ext v19.16b, v16.16b, v16.16b, #6 ext v21.16b, v20.16b, v20.16b, #2 ext v22.16b, v20.16b, v20.16b, #4 ext v23.16b, v20.16b, v20.16b, #6 subs \h, \h, #2 smull v16.4s, v16.4h, v0.h[0] smlal v16.4s, v17.4h, v0.h[1] smlal v16.4s, v18.4h, v0.h[2] smlal v16.4s, v19.4h, v0.h[3] smull v20.4s, v20.4h, v0.h[0] smlal v20.4s, v21.4h, v0.h[1] smlal v20.4s, v22.4h, v0.h[2] smlal v20.4s, v23.4h, v0.h[3] srshl v16.4s, v16.4s, v30.4s // -(6-intermediate_bits) srshl v20.4s, v20.4s, v30.4s // -(6-intermediate_bits) .ifc \type, put sqxtun v16.4h, v16.4s sqxtun2 v16.8h, v20.4s srshl v16.8h, v16.8h, v29.8h // -intermediate_bits umin v16.8h, v16.8h, v31.8h .else uzp1 v16.8h, v16.8h, v20.8h // Same as xtn, xtn2 sub v16.8h, v16.8h, v28.8h // PREP_BIAS .endif st1 {v16.d}[0], [\dst], \d_strd st1 {v16.d}[1], [\ds2], \d_strd b.gt 4b ret 80: 160: 320: 640: 1280: // 8xN, 16xN, 32xN, ... h AARCH64_VALID_JUMP_TARGET ld1 {v0.8b}, [\xmx] sub \src, \src, #6 add \ds2, \dst, \d_strd add \sr2, \src, \s_strd lsl \s_strd, \s_strd, #1 sxtl v0.8h, v0.8b sub \s_strd, \s_strd, \w, uxtw #1 sub \s_strd, \s_strd, #16 .ifc \type, put lsl \d_strd, \d_strd, #1 sub \d_strd, \d_strd, \w, uxtw #1 .endif 81: ld1 {v16.8h, v17.8h}, [\src], #32 ld1 {v20.8h, v21.8h}, [\sr2], #32 mov \mx, \w 8: smull v18.4s, v16.4h, v0.h[0] smull2 v19.4s, v16.8h, v0.h[0] smull v22.4s, v20.4h, v0.h[0] smull2 v23.4s, v20.8h, v0.h[0] .irpc i, 1234567 ext v24.16b, v16.16b, v17.16b, #(2*\i) ext v25.16b, v20.16b, v21.16b, #(2*\i) smlal v18.4s, v24.4h, v0.h[\i] smlal2 v19.4s, v24.8h, v0.h[\i] smlal v22.4s, v25.4h, v0.h[\i] smlal2 v23.4s, v25.8h, v0.h[\i] .endr subs \mx, \mx, #8 srshl v18.4s, v18.4s, v30.4s // -(6-intermediate_bits) srshl v19.4s, v19.4s, v30.4s // -(6-intermediate_bits) srshl v22.4s, v22.4s, v30.4s // -(6-intermediate_bits) srshl v23.4s, v23.4s, v30.4s // -(6-intermediate_bits) .ifc \type, put sqxtun v18.4h, v18.4s sqxtun2 v18.8h, v19.4s sqxtun v22.4h, v22.4s sqxtun2 v22.8h, v23.4s srshl v18.8h, v18.8h, v29.8h // -intermediate_bits srshl v22.8h, v22.8h, v29.8h // -intermediate_bits umin v18.8h, v18.8h, v31.8h umin v22.8h, v22.8h, v31.8h .else uzp1 v18.8h, v18.8h, v19.8h // Same as xtn, xtn2 uzp1 v22.8h, v22.8h, v23.8h // Ditto sub v18.8h, v18.8h, v28.8h // PREP_BIAS sub v22.8h, v22.8h, v28.8h // PREP_BIAS .endif st1 {v18.8h}, [\dst], #16 st1 {v22.8h}, [\ds2], #16 b.le 9f mov v16.16b, v17.16b mov v20.16b, v21.16b ld1 {v17.8h}, [\src], #16 ld1 {v21.8h}, [\sr2], #16 b 8b 9: add \dst, \dst, \d_strd add \ds2, \ds2, \d_strd add \src, \src, \s_strd add \sr2, \sr2, \s_strd subs \h, \h, #2 b.gt 81b ret L(\type\()_8tap_h_tbl): .hword L(\type\()_8tap_h_tbl) - 1280b .hword L(\type\()_8tap_h_tbl) - 640b .hword L(\type\()_8tap_h_tbl) - 320b .hword L(\type\()_8tap_h_tbl) - 160b .hword L(\type\()_8tap_h_tbl) - 80b .hword L(\type\()_8tap_h_tbl) - 40b .hword L(\type\()_8tap_h_tbl) - 20b .hword 0 L(\type\()_8tap_v): cmp \h, #4 ubfx w10, \my, #7, #7 and \my, \my, #0x7f b.le 4f mov \my, w10 4: add \xmy, x11, \my, uxtw #3 .ifc \type, prep dup v30.4s, w12 // 6 - intermediate_bits movi v29.8h, #(PREP_BIAS >> 8), lsl #8 .endif adr x10, L(\type\()_8tap_v_tbl) ldrh w9, [x10, x9, lsl #1] .ifc \type, prep neg v30.4s, v30.4s // -(6-intermediate_bits) .endif sub x10, x10, w9, uxtw br x10 20: // 2xN v AARCH64_VALID_JUMP_TARGET .ifc \type, put b.gt 28f cmp \h, #2 add \xmy, \xmy, #2 ld1 {v0.s}[0], [\xmy] sub \src, \src, \s_strd add \ds2, \dst, \d_strd add \sr2, \src, \s_strd lsl \s_strd, \s_strd, #1 lsl \d_strd, \d_strd, #1 sxtl v0.8h, v0.8b // 2x2 v load_s \src, \sr2, \s_strd, v1, v2, v3, v4, v5 interleave_1_s v1, v2, v3, v4, v5 b.gt 24f smull_smlal_4 v6, v1, v2, v3, v4 sqrshrun_h 6, v6 umin_h v31, .8h, v6 st_s \d_strd, v6, 2 ret 24: // 2x4 v load_s \sr2, \src, \s_strd, v6, v7 interleave_1_s v5, v6, v7 smull_smlal_4 v16, v1, v2, v3, v4 smull_smlal_4 v17, v3, v4, v5, v6 sqrshrun_h 6, v16, v17 umin_h v31, .8h, v16 st_s \d_strd, v16, 4 ret 28: // 2x6, 2x8, 2x12, 2x16 v ld1 {v0.8b}, [\xmy] sub \sr2, \src, \s_strd, lsl #1 add \ds2, \dst, \d_strd sub \src, \sr2, \s_strd lsl \d_strd, \d_strd, #1 lsl \s_strd, \s_strd, #1 sxtl v0.8h, v0.8b load_s \src, \sr2, \s_strd, v1, v2, v3, v4, v5, v6, v7 interleave_1_s v1, v2, v3, v4, v5 interleave_1_s v5, v6, v7 216: subs \h, \h, #4 load_s \sr2, \src, \s_strd, v16, v17, v18, v19 interleave_1_s v7, v16, v17, v18, v19 smull_smlal_8 v24, v1, v2, v3, v4, v5, v6, v7, v16 smull_smlal_8 v25, v3, v4, v5, v6, v7, v16, v17, v18 sqrshrun_h 6, v24, v25 umin_h v31, .8h, v24 st_s \d_strd, v24, 4 b.le 0f cmp \h, #2 mov v1.16b, v5.16b mov v2.16b, v6.16b mov v3.16b, v7.16b mov v4.16b, v16.16b mov v5.16b, v17.16b mov v6.16b, v18.16b mov v7.16b, v19.16b b.eq 26f b 216b 26: load_s \sr2, \src, \s_strd, v16, v17 interleave_1_s v7, v16, v17 smull_smlal_8 v24, v1, v2, v3, v4, v5, v6, v7, v16 sqrshrun_h 6, v24 umin_h v31, .4h, v24 st_s \d_strd, v24, 2 0: ret .endif 40: AARCH64_VALID_JUMP_TARGET b.gt 480f // 4x2, 4x4 v cmp \h, #2 add \xmy, \xmy, #2 ld1 {v0.s}[0], [\xmy] sub \src, \src, \s_strd add \ds2, \dst, \d_strd add \sr2, \src, \s_strd lsl \s_strd, \s_strd, #1 lsl \d_strd, \d_strd, #1 sxtl v0.8h, v0.8b load_4h \src, \sr2, \s_strd, v1, v2, v3, v4, v5 smull_smlal_4 v6, v1, v2, v3, v4 smull_smlal_4 v7, v2, v3, v4, v5 shift_store_4 \type, \d_strd, v6, v7 b.le 0f load_4h \sr2, \src, \s_strd, v6, v7 smull_smlal_4 v1, v3, v4, v5, v6 smull_smlal_4 v2, v4, v5, v6, v7 shift_store_4 \type, \d_strd, v1, v2 0: ret 480: // 4x6, 4x8, 4x12, 4x16 v ld1 {v0.8b}, [\xmy] sub \sr2, \src, \s_strd, lsl #1 add \ds2, \dst, \d_strd sub \src, \sr2, \s_strd lsl \s_strd, \s_strd, #1 lsl \d_strd, \d_strd, #1 sxtl v0.8h, v0.8b load_4h \src, \sr2, \s_strd, v16, v17, v18, v19, v20, v21, v22 48: subs \h, \h, #4 load_4h \sr2, \src, \s_strd, v23, v24, v25, v26 smull_smlal_8 v1, v16, v17, v18, v19, v20, v21, v22, v23 smull_smlal_8 v2, v17, v18, v19, v20, v21, v22, v23, v24 smull_smlal_8 v3, v18, v19, v20, v21, v22, v23, v24, v25 smull_smlal_8 v4, v19, v20, v21, v22, v23, v24, v25, v26 shift_store_4 \type, \d_strd, v1, v2, v3, v4 b.le 0f cmp \h, #2 mov v16.8b, v20.8b mov v17.8b, v21.8b mov v18.8b, v22.8b mov v19.8b, v23.8b mov v20.8b, v24.8b mov v21.8b, v25.8b mov v22.8b, v26.8b b.eq 46f b 48b 46: load_4h \sr2, \src, \s_strd, v23, v24 smull_smlal_8 v1, v16, v17, v18, v19, v20, v21, v22, v23 smull_smlal_8 v2, v17, v18, v19, v20, v21, v22, v23, v24 shift_store_4 \type, \d_strd, v1, v2 0: ret 80: AARCH64_VALID_JUMP_TARGET b.gt 880f // 8x2, 8x4 v cmp \h, #2 add \xmy, \xmy, #2 ld1 {v0.s}[0], [\xmy] sub \src, \src, \s_strd add \ds2, \dst, \d_strd add \sr2, \src, \s_strd lsl \s_strd, \s_strd, #1 lsl \d_strd, \d_strd, #1 sxtl v0.8h, v0.8b load_8h \src, \sr2, \s_strd, v1, v2, v3, v4, v5 smull_smlal_4 v16, v1, v2, v3, v4 smull2_smlal2_4 v17, v1, v2, v3, v4 smull_smlal_4 v18, v2, v3, v4, v5 smull2_smlal2_4 v19, v2, v3, v4, v5 shift_store_8 \type, \d_strd, v16, v17, v18, v19 b.le 0f load_8h \sr2, \src, \s_strd, v6, v7 smull_smlal_4 v16, v3, v4, v5, v6 smull2_smlal2_4 v17, v3, v4, v5, v6 smull_smlal_4 v18, v4, v5, v6, v7 smull2_smlal2_4 v19, v4, v5, v6, v7 shift_store_8 \type, \d_strd, v16, v17, v18, v19 0: ret 880: // 8x6, 8x8, 8x16, 8x32 v 1680: // 16x8, 16x16, ... 320: // 32x8, 32x16, ... 640: 1280: AARCH64_VALID_JUMP_TARGET ld1 {v0.8b}, [\xmy] sub \src, \src, \s_strd sub \src, \src, \s_strd, lsl #1 sxtl v0.8h, v0.8b mov \my, \h 168: add \ds2, \dst, \d_strd add \sr2, \src, \s_strd lsl \s_strd, \s_strd, #1 lsl \d_strd, \d_strd, #1 load_8h \src, \sr2, \s_strd, v16, v17, v18, v19, v20, v21, v22 88: subs \h, \h, #2 load_8h \sr2, \src, \s_strd, v23, v24 smull_smlal_8 v1, v16, v17, v18, v19, v20, v21, v22, v23 smull2_smlal2_8 v2, v16, v17, v18, v19, v20, v21, v22, v23 smull_smlal_8 v3, v17, v18, v19, v20, v21, v22, v23, v24 smull2_smlal2_8 v4, v17, v18, v19, v20, v21, v22, v23, v24 shift_store_8 \type, \d_strd, v1, v2, v3, v4 b.le 9f subs \h, \h, #2 load_8h \sr2, \src, \s_strd, v25, v26 smull_smlal_8 v1, v18, v19, v20, v21, v22, v23, v24, v25 smull2_smlal2_8 v2, v18, v19, v20, v21, v22, v23, v24, v25 smull_smlal_8 v3, v19, v20, v21, v22, v23, v24, v25, v26 smull2_smlal2_8 v4, v19, v20, v21, v22, v23, v24, v25, v26 shift_store_8 \type, \d_strd, v1, v2, v3, v4 b.le 9f mov v16.16b, v20.16b mov v17.16b, v21.16b mov v18.16b, v22.16b mov v19.16b, v23.16b mov v20.16b, v24.16b mov v21.16b, v25.16b mov v22.16b, v26.16b b 88b 9: subs \w, \w, #8 b.le 0f asr \s_strd, \s_strd, #1 asr \d_strd, \d_strd, #1 msub \src, \s_strd, \xmy, \src msub \dst, \d_strd, \xmy, \dst sub \src, \src, \s_strd, lsl #3 mov \h, \my add \src, \src, #16 add \dst, \dst, #16 b 168b 0: ret 160: AARCH64_VALID_JUMP_TARGET b.gt 1680b // 16x2, 16x4 v add \xmy, \xmy, #2 ld1 {v0.s}[0], [\xmy] sub \src, \src, \s_strd sxtl v0.8h, v0.8b load_16h \src, \src, \s_strd, v16, v17, v18, v19, v20, v21 16: load_16h \src, \src, \s_strd, v22, v23 subs \h, \h, #1 smull_smlal_4 v1, v16, v18, v20, v22 smull2_smlal2_4 v2, v16, v18, v20, v22 smull_smlal_4 v3, v17, v19, v21, v23 smull2_smlal2_4 v4, v17, v19, v21, v23 shift_store_16 \type, \d_strd, x0, v1, v2, v3, v4 b.le 0f mov v16.16b, v18.16b mov v17.16b, v19.16b mov v18.16b, v20.16b mov v19.16b, v21.16b mov v20.16b, v22.16b mov v21.16b, v23.16b b 16b 0: ret L(\type\()_8tap_v_tbl): .hword L(\type\()_8tap_v_tbl) - 1280b .hword L(\type\()_8tap_v_tbl) - 640b .hword L(\type\()_8tap_v_tbl) - 320b .hword L(\type\()_8tap_v_tbl) - 160b .hword L(\type\()_8tap_v_tbl) - 80b .hword L(\type\()_8tap_v_tbl) - 40b .hword L(\type\()_8tap_v_tbl) - 20b .hword 0 L(\type\()_8tap_hv): cmp \h, #4 ubfx w10, \my, #7, #7 and \my, \my, #0x7f b.le 4f mov \my, w10 4: add \xmy, x11, \my, uxtw #3 adr x10, L(\type\()_8tap_hv_tbl) dup v30.4s, w12 // 6 - intermediate_bits ldrh w9, [x10, x9, lsl #1] neg v30.4s, v30.4s // -(6-intermediate_bits) .ifc \type, put dup v29.4s, w13 // 6 + intermediate_bits .else movi v29.8h, #(PREP_BIAS >> 8), lsl #8 .endif sub x10, x10, w9, uxtw .ifc \type, put neg v29.4s, v29.4s // -(6+intermediate_bits) .endif br x10 20: AARCH64_VALID_JUMP_TARGET .ifc \type, put add \xmx, \xmx, #2 ld1 {v0.s}[0], [\xmx] b.gt 280f add \xmy, \xmy, #2 ld1 {v1.s}[0], [\xmy] // 2x2, 2x4 hv sub \sr2, \src, #2 sub \src, \sr2, \s_strd add \ds2, \dst, \d_strd lsl \s_strd, \s_strd, #1 lsl \d_strd, \d_strd, #1 sxtl v0.8h, v0.8b sxtl v1.8h, v1.8b mov x15, x30 ld1 {v27.8h}, [\src], \s_strd ext v28.16b, v27.16b, v27.16b, #2 smull v27.4s, v27.4h, v0.4h smull v28.4s, v28.4h, v0.4h addp v27.4s, v27.4s, v28.4s addp v16.4s, v27.4s, v27.4s srshl v16.2s, v16.2s, v30.2s // -(6-intermediate_bits) bl L(\type\()_8tap_filter_2) // The intermediates from the horizontal pass fit in 16 bit without // any bias; we could just as well keep them as .4s, but narrowing // them to .4h gives a significant speedup on out of order cores // (at the cost of a smaller slowdown on in-order cores such as A53). xtn v16.4h, v16.4s trn1 v16.2s, v16.2s, v24.2s mov v17.8b, v24.8b 2: bl L(\type\()_8tap_filter_2) ext v18.8b, v17.8b, v24.8b, #4 smull v2.4s, v16.4h, v1.h[0] smlal v2.4s, v17.4h, v1.h[1] smlal v2.4s, v18.4h, v1.h[2] smlal v2.4s, v24.4h, v1.h[3] srshl v2.4s, v2.4s, v29.4s // -(6+intermediate_bits) sqxtun v2.4h, v2.4s umin v2.4h, v2.4h, v31.4h subs \h, \h, #2 st1 {v2.s}[0], [\dst], \d_strd st1 {v2.s}[1], [\ds2], \d_strd b.le 0f mov v16.8b, v18.8b mov v17.8b, v24.8b b 2b 280: // 2x8, 2x16, 2x32 hv ld1 {v1.8b}, [\xmy] sub \src, \src, #2 sub \sr2, \src, \s_strd, lsl #1 sub \src, \sr2, \s_strd add \ds2, \dst, \d_strd lsl \s_strd, \s_strd, #1 lsl \d_strd, \d_strd, #1 sxtl v0.8h, v0.8b sxtl v1.8h, v1.8b mov x15, x30 ld1 {v27.8h}, [\src], \s_strd ext v28.16b, v27.16b, v27.16b, #2 smull v27.4s, v27.4h, v0.4h smull v28.4s, v28.4h, v0.4h addp v27.4s, v27.4s, v28.4s addp v16.4s, v27.4s, v27.4s srshl v16.2s, v16.2s, v30.2s // -(6-intermediate_bits) // The intermediates from the horizontal pass fit in 16 bit without // any bias; we could just as well keep them as .4s, but narrowing // them to .4h gives a significant speedup on out of order cores // (at the cost of a smaller slowdown on in-order cores such as A53). bl L(\type\()_8tap_filter_2) xtn v16.4h, v16.4s trn1 v16.2s, v16.2s, v24.2s mov v17.8b, v24.8b bl L(\type\()_8tap_filter_2) ext v18.8b, v17.8b, v24.8b, #4 mov v19.8b, v24.8b bl L(\type\()_8tap_filter_2) ext v20.8b, v19.8b, v24.8b, #4 mov v21.8b, v24.8b 28: bl L(\type\()_8tap_filter_2) ext v22.8b, v21.8b, v24.8b, #4 smull v3.4s, v16.4h, v1.h[0] smlal v3.4s, v17.4h, v1.h[1] smlal v3.4s, v18.4h, v1.h[2] smlal v3.4s, v19.4h, v1.h[3] smlal v3.4s, v20.4h, v1.h[4] smlal v3.4s, v21.4h, v1.h[5] smlal v3.4s, v22.4h, v1.h[6] smlal v3.4s, v24.4h, v1.h[7] srshl v3.4s, v3.4s, v29.4s // -(6+intermediate_bits) sqxtun v3.4h, v3.4s umin v3.4h, v3.4h, v31.4h subs \h, \h, #2 st1 {v3.s}[0], [\dst], \d_strd st1 {v3.s}[1], [\ds2], \d_strd b.le 0f mov v16.8b, v18.8b mov v17.8b, v19.8b mov v18.8b, v20.8b mov v19.8b, v21.8b mov v20.8b, v22.8b mov v21.8b, v24.8b b 28b 0: ret x15 L(\type\()_8tap_filter_2): ld1 {v25.8h}, [\sr2], \s_strd ld1 {v27.8h}, [\src], \s_strd ext v26.16b, v25.16b, v25.16b, #2 ext v28.16b, v27.16b, v27.16b, #2 trn1 v24.2s, v25.2s, v27.2s trn2 v27.2s, v25.2s, v27.2s trn1 v25.2s, v26.2s, v28.2s trn2 v28.2s, v26.2s, v28.2s smull v24.4s, v24.4h, v0.h[0] smlal v24.4s, v25.4h, v0.h[1] smlal v24.4s, v27.4h, v0.h[2] smlal v24.4s, v28.4h, v0.h[3] srshl v24.4s, v24.4s, v30.4s // -(6-intermediate_bits) xtn v24.4h, v24.4s ret .endif 40: AARCH64_VALID_JUMP_TARGET add \xmx, \xmx, #2 ld1 {v0.s}[0], [\xmx] b.gt 480f add \xmy, \xmy, #2 ld1 {v1.s}[0], [\xmy] sub \sr2, \src, #2 sub \src, \sr2, \s_strd add \ds2, \dst, \d_strd lsl \s_strd, \s_strd, #1 lsl \d_strd, \d_strd, #1 sxtl v0.8h, v0.8b sxtl v1.8h, v1.8b mov x15, x30 // 4x2, 4x4 hv ld1 {v25.8h}, [\src], \s_strd ext v26.16b, v25.16b, v25.16b, #2 ext v27.16b, v25.16b, v25.16b, #4 ext v28.16b, v25.16b, v25.16b, #6 smull v25.4s, v25.4h, v0.h[0] smlal v25.4s, v26.4h, v0.h[1] smlal v25.4s, v27.4h, v0.h[2] smlal v25.4s, v28.4h, v0.h[3] srshl v16.4s, v25.4s, v30.4s // -(6-intermediate_bits) // The intermediates from the horizontal pass fit in 16 bit without // any bias; we could just as well keep them as .4s, but narrowing // them to .4h gives a significant speedup on out of order cores // (at the cost of a smaller slowdown on in-order cores such as A53). xtn v16.4h, v16.4s bl L(\type\()_8tap_filter_4) mov v17.8b, v24.8b mov v18.8b, v25.8b 4: bl L(\type\()_8tap_filter_4) smull v2.4s, v16.4h, v1.h[0] smlal v2.4s, v17.4h, v1.h[1] smlal v2.4s, v18.4h, v1.h[2] smlal v2.4s, v24.4h, v1.h[3] smull v3.4s, v17.4h, v1.h[0] smlal v3.4s, v18.4h, v1.h[1] smlal v3.4s, v24.4h, v1.h[2] smlal v3.4s, v25.4h, v1.h[3] .ifc \type, put srshl v2.4s, v2.4s, v29.4s // -(6+intermediate_bits) srshl v3.4s, v3.4s, v29.4s // -(6+intermediate_bits) sqxtun v2.4h, v2.4s sqxtun2 v2.8h, v3.4s umin v2.8h, v2.8h, v31.8h .else rshrn v2.4h, v2.4s, #6 rshrn2 v2.8h, v3.4s, #6 sub v2.8h, v2.8h, v29.8h // PREP_BIAS .endif subs \h, \h, #2 st1 {v2.d}[0], [\dst], \d_strd st1 {v2.d}[1], [\ds2], \d_strd b.le 0f mov v16.8b, v18.8b mov v17.8b, v24.8b mov v18.8b, v25.8b b 4b 480: // 4x8, 4x16, 4x32 hv ld1 {v1.8b}, [\xmy] sub \src, \src, #2 sub \sr2, \src, \s_strd, lsl #1 sub \src, \sr2, \s_strd add \ds2, \dst, \d_strd lsl \s_strd, \s_strd, #1 lsl \d_strd, \d_strd, #1 sxtl v0.8h, v0.8b sxtl v1.8h, v1.8b mov x15, x30 ld1 {v25.8h}, [\src], \s_strd ext v26.16b, v25.16b, v25.16b, #2 ext v27.16b, v25.16b, v25.16b, #4 ext v28.16b, v25.16b, v25.16b, #6 smull v25.4s, v25.4h, v0.h[0] smlal v25.4s, v26.4h, v0.h[1] smlal v25.4s, v27.4h, v0.h[2] smlal v25.4s, v28.4h, v0.h[3] srshl v16.4s, v25.4s, v30.4s // -(6-intermediate_bits) // The intermediates from the horizontal pass fit in 16 bit without // any bias; we could just as well keep them as .4s, but narrowing // them to .4h gives a significant speedup on out of order cores // (at the cost of a smaller slowdown on in-order cores such as A53). xtn v16.4h, v16.4s bl L(\type\()_8tap_filter_4) mov v17.8b, v24.8b mov v18.8b, v25.8b bl L(\type\()_8tap_filter_4) mov v19.8b, v24.8b mov v20.8b, v25.8b bl L(\type\()_8tap_filter_4) mov v21.8b, v24.8b mov v22.8b, v25.8b 48: bl L(\type\()_8tap_filter_4) smull v3.4s, v16.4h, v1.h[0] smlal v3.4s, v17.4h, v1.h[1] smlal v3.4s, v18.4h, v1.h[2] smlal v3.4s, v19.4h, v1.h[3] smlal v3.4s, v20.4h, v1.h[4] smlal v3.4s, v21.4h, v1.h[5] smlal v3.4s, v22.4h, v1.h[6] smlal v3.4s, v24.4h, v1.h[7] smull v4.4s, v17.4h, v1.h[0] smlal v4.4s, v18.4h, v1.h[1] smlal v4.4s, v19.4h, v1.h[2] smlal v4.4s, v20.4h, v1.h[3] smlal v4.4s, v21.4h, v1.h[4] smlal v4.4s, v22.4h, v1.h[5] smlal v4.4s, v24.4h, v1.h[6] smlal v4.4s, v25.4h, v1.h[7] .ifc \type, put srshl v3.4s, v3.4s, v29.4s // -(6+intermediate_bits) srshl v4.4s, v4.4s, v29.4s // -(6+intermediate_bits) sqxtun v3.4h, v3.4s sqxtun2 v3.8h, v4.4s umin v3.8h, v3.8h, v31.8h .else rshrn v3.4h, v3.4s, #6 rshrn2 v3.8h, v4.4s, #6 sub v3.8h, v3.8h, v29.8h // PREP_BIAS .endif subs \h, \h, #2 st1 {v3.d}[0], [\dst], \d_strd st1 {v3.d}[1], [\ds2], \d_strd b.le 0f mov v16.8b, v18.8b mov v17.8b, v19.8b mov v18.8b, v20.8b mov v19.8b, v21.8b mov v20.8b, v22.8b mov v21.8b, v24.8b mov v22.8b, v25.8b b 48b 0: ret x15 L(\type\()_8tap_filter_4): ld1 {v24.8h}, [\sr2], \s_strd ld1 {v25.8h}, [\src], \s_strd ext v26.16b, v24.16b, v24.16b, #2 ext v27.16b, v24.16b, v24.16b, #4 ext v28.16b, v24.16b, v24.16b, #6 smull v24.4s, v24.4h, v0.h[0] smlal v24.4s, v26.4h, v0.h[1] smlal v24.4s, v27.4h, v0.h[2] smlal v24.4s, v28.4h, v0.h[3] ext v26.16b, v25.16b, v25.16b, #2 ext v27.16b, v25.16b, v25.16b, #4 ext v28.16b, v25.16b, v25.16b, #6 smull v25.4s, v25.4h, v0.h[0] smlal v25.4s, v26.4h, v0.h[1] smlal v25.4s, v27.4h, v0.h[2] smlal v25.4s, v28.4h, v0.h[3] srshl v24.4s, v24.4s, v30.4s // -(6-intermediate_bits) srshl v25.4s, v25.4s, v30.4s // -(6-intermediate_bits) xtn v24.4h, v24.4s xtn v25.4h, v25.4s ret 80: 160: 320: AARCH64_VALID_JUMP_TARGET b.gt 880f add \xmy, \xmy, #2 ld1 {v0.8b}, [\xmx] ld1 {v1.s}[0], [\xmy] sub \src, \src, #6 sub \src, \src, \s_strd sxtl v0.8h, v0.8b sxtl v1.8h, v1.8b mov x15, x30 mov \my, \h 164: // 8x2, 8x4, 16x2, 16x4, 32x2, 32x4 hv add \ds2, \dst, \d_strd add \sr2, \src, \s_strd lsl \d_strd, \d_strd, #1 lsl \s_strd, \s_strd, #1 ld1 {v27.8h, v28.8h}, [\src], \s_strd smull v24.4s, v27.4h, v0.h[0] smull2 v25.4s, v27.8h, v0.h[0] .irpc i, 1234567 ext v26.16b, v27.16b, v28.16b, #(2*\i) smlal v24.4s, v26.4h, v0.h[\i] smlal2 v25.4s, v26.8h, v0.h[\i] .endr srshl v24.4s, v24.4s, v30.4s // -(6-intermediate_bits) srshl v25.4s, v25.4s, v30.4s // -(6-intermediate_bits) // The intermediates from the horizontal pass fit in 16 bit without // any bias; we could just as well keep them as .4s, but narrowing // them to .4h gives a significant speedup on out of order cores // (at the cost of a smaller slowdown on in-order cores such as A53), // and conserves register space (no need to clobber v8-v15). uzp1 v16.8h, v24.8h, v25.8h // Same as xtn, xtn2 bl L(\type\()_8tap_filter_8) mov v17.16b, v23.16b mov v18.16b, v24.16b 8: smull v2.4s, v16.4h, v1.h[0] smull2 v3.4s, v16.8h, v1.h[0] bl L(\type\()_8tap_filter_8) smull v4.4s, v17.4h, v1.h[0] smull2 v5.4s, v17.8h, v1.h[0] smlal v2.4s, v17.4h, v1.h[1] smlal2 v3.4s, v17.8h, v1.h[1] smlal v4.4s, v18.4h, v1.h[1] smlal2 v5.4s, v18.8h, v1.h[1] smlal v2.4s, v18.4h, v1.h[2] smlal2 v3.4s, v18.8h, v1.h[2] smlal v4.4s, v23.4h, v1.h[2] smlal2 v5.4s, v23.8h, v1.h[2] smlal v2.4s, v23.4h, v1.h[3] smlal2 v3.4s, v23.8h, v1.h[3] smlal v4.4s, v24.4h, v1.h[3] smlal2 v5.4s, v24.8h, v1.h[3] .ifc \type, put srshl v2.4s, v2.4s, v29.4s // -(6+intermediate_bits) srshl v3.4s, v3.4s, v29.4s // -(6+intermediate_bits) srshl v4.4s, v4.4s, v29.4s // -(6+intermediate_bits) srshl v5.4s, v5.4s, v29.4s // -(6+intermediate_bits) sqxtun v2.4h, v2.4s sqxtun2 v2.8h, v3.4s sqxtun v3.4h, v4.4s sqxtun2 v3.8h, v5.4s umin v2.8h, v2.8h, v31.8h umin v3.8h, v3.8h, v31.8h .else rshrn v2.4h, v2.4s, #6 rshrn2 v2.8h, v3.4s, #6 rshrn v3.4h, v4.4s, #6 rshrn2 v3.8h, v5.4s, #6 sub v2.8h, v2.8h, v29.8h // PREP_BIAS sub v3.8h, v3.8h, v29.8h // PREP_BIAS .endif subs \h, \h, #2 st1 {v2.8h}, [\dst], \d_strd st1 {v3.8h}, [\ds2], \d_strd b.le 9f mov v16.16b, v18.16b mov v17.16b, v23.16b mov v18.16b, v24.16b b 8b 9: subs \w, \w, #8 b.le 0f asr \s_strd, \s_strd, #1 asr \d_strd, \d_strd, #1 msub \src, \s_strd, \xmy, \src msub \dst, \d_strd, \xmy, \dst sub \src, \src, \s_strd, lsl #2 mov \h, \my add \src, \src, #16 add \dst, \dst, #16 b 164b 880: // 8x8, 8x16, ..., 16x8, ..., 32x8, ... hv 640: 1280: AARCH64_VALID_JUMP_TARGET ld1 {v0.8b}, [\xmx] ld1 {v1.8b}, [\xmy] sub \src, \src, #6 sub \src, \src, \s_strd sub \src, \src, \s_strd, lsl #1 sxtl v0.8h, v0.8b sxtl v1.8h, v1.8b mov x15, x30 mov \my, \h 168: add \ds2, \dst, \d_strd add \sr2, \src, \s_strd lsl \d_strd, \d_strd, #1 lsl \s_strd, \s_strd, #1 ld1 {v27.8h, v28.8h}, [\src], \s_strd smull v24.4s, v27.4h, v0.h[0] smull2 v25.4s, v27.8h, v0.h[0] .irpc i, 1234567 ext v26.16b, v27.16b, v28.16b, #(2*\i) smlal v24.4s, v26.4h, v0.h[\i] smlal2 v25.4s, v26.8h, v0.h[\i] .endr srshl v24.4s, v24.4s, v30.4s // -(6-intermediate_bits) srshl v25.4s, v25.4s, v30.4s // -(6-intermediate_bits) // The intermediates from the horizontal pass fit in 16 bit without // any bias; we could just as well keep them as .4s, but narrowing // them to .4h gives a significant speedup on out of order cores // (at the cost of a smaller slowdown on in-order cores such as A53), // and conserves register space (no need to clobber v8-v15). uzp1 v16.8h, v24.8h, v25.8h // Same as xtn, xtn2 bl L(\type\()_8tap_filter_8) mov v17.16b, v23.16b mov v18.16b, v24.16b bl L(\type\()_8tap_filter_8) mov v19.16b, v23.16b mov v20.16b, v24.16b bl L(\type\()_8tap_filter_8) mov v21.16b, v23.16b mov v22.16b, v24.16b 88: smull v2.4s, v16.4h, v1.h[0] smull2 v3.4s, v16.8h, v1.h[0] bl L(\type\()_8tap_filter_8) smull v4.4s, v17.4h, v1.h[0] smull2 v5.4s, v17.8h, v1.h[0] smlal v2.4s, v17.4h, v1.h[1] smlal2 v3.4s, v17.8h, v1.h[1] smlal v4.4s, v18.4h, v1.h[1] smlal2 v5.4s, v18.8h, v1.h[1] smlal v2.4s, v18.4h, v1.h[2] smlal2 v3.4s, v18.8h, v1.h[2] smlal v4.4s, v19.4h, v1.h[2] smlal2 v5.4s, v19.8h, v1.h[2] smlal v2.4s, v19.4h, v1.h[3] smlal2 v3.4s, v19.8h, v1.h[3] smlal v4.4s, v20.4h, v1.h[3] smlal2 v5.4s, v20.8h, v1.h[3] smlal v2.4s, v20.4h, v1.h[4] smlal2 v3.4s, v20.8h, v1.h[4] smlal v4.4s, v21.4h, v1.h[4] smlal2 v5.4s, v21.8h, v1.h[4] smlal v2.4s, v21.4h, v1.h[5] smlal2 v3.4s, v21.8h, v1.h[5] smlal v4.4s, v22.4h, v1.h[5] smlal2 v5.4s, v22.8h, v1.h[5] smlal v2.4s, v22.4h, v1.h[6] smlal2 v3.4s, v22.8h, v1.h[6] smlal v4.4s, v23.4h, v1.h[6] smlal2 v5.4s, v23.8h, v1.h[6] smlal v2.4s, v23.4h, v1.h[7] smlal2 v3.4s, v23.8h, v1.h[7] smlal v4.4s, v24.4h, v1.h[7] smlal2 v5.4s, v24.8h, v1.h[7] .ifc \type, put srshl v2.4s, v2.4s, v29.4s // -(6+intermediate_bits) srshl v3.4s, v3.4s, v29.4s // -(6+intermediate_bits) srshl v4.4s, v4.4s, v29.4s // -(6+intermediate_bits) srshl v5.4s, v5.4s, v29.4s // -(6+intermediate_bits) sqxtun v2.4h, v2.4s sqxtun2 v2.8h, v3.4s sqxtun v3.4h, v4.4s sqxtun2 v3.8h, v5.4s umin v2.8h, v2.8h, v31.8h umin v3.8h, v3.8h, v31.8h .else rshrn v2.4h, v2.4s, #6 rshrn2 v2.8h, v3.4s, #6 rshrn v3.4h, v4.4s, #6 rshrn2 v3.8h, v5.4s, #6 sub v2.8h, v2.8h, v29.8h // PREP_BIAS sub v3.8h, v3.8h, v29.8h // PREP_BIAS .endif subs \h, \h, #2 st1 {v2.8h}, [\dst], \d_strd st1 {v3.8h}, [\ds2], \d_strd b.le 9f mov v16.16b, v18.16b mov v17.16b, v19.16b mov v18.16b, v20.16b mov v19.16b, v21.16b mov v20.16b, v22.16b mov v21.16b, v23.16b mov v22.16b, v24.16b b 88b 9: subs \w, \w, #8 b.le 0f asr \s_strd, \s_strd, #1 asr \d_strd, \d_strd, #1 msub \src, \s_strd, \xmy, \src msub \dst, \d_strd, \xmy, \dst sub \src, \src, \s_strd, lsl #3 mov \h, \my add \src, \src, #16 add \dst, \dst, #16 b 168b 0: ret x15 L(\type\()_8tap_filter_8): ld1 {v4.8h, v5.8h}, [\sr2], \s_strd ld1 {v6.8h, v7.8h}, [\src], \s_strd smull v25.4s, v4.4h, v0.h[0] smull2 v26.4s, v4.8h, v0.h[0] smull v27.4s, v6.4h, v0.h[0] smull2 v28.4s, v6.8h, v0.h[0] .irpc i, 1234567 ext v23.16b, v4.16b, v5.16b, #(2*\i) ext v24.16b, v6.16b, v7.16b, #(2*\i) smlal v25.4s, v23.4h, v0.h[\i] smlal2 v26.4s, v23.8h, v0.h[\i] smlal v27.4s, v24.4h, v0.h[\i] smlal2 v28.4s, v24.8h, v0.h[\i] .endr srshl v25.4s, v25.4s, v30.4s // -(6-intermediate_bits) srshl v26.4s, v26.4s, v30.4s // -(6-intermediate_bits) srshl v27.4s, v27.4s, v30.4s // -(6-intermediate_bits) srshl v28.4s, v28.4s, v30.4s // -(6-intermediate_bits) uzp1 v23.8h, v25.8h, v26.8h // Same as xtn, xtn2 uzp1 v24.8h, v27.8h, v28.8h // Ditto ret L(\type\()_8tap_hv_tbl): .hword L(\type\()_8tap_hv_tbl) - 1280b .hword L(\type\()_8tap_hv_tbl) - 640b .hword L(\type\()_8tap_hv_tbl) - 320b .hword L(\type\()_8tap_hv_tbl) - 160b .hword L(\type\()_8tap_hv_tbl) - 80b .hword L(\type\()_8tap_hv_tbl) - 40b .hword L(\type\()_8tap_hv_tbl) - 20b .hword 0 endfunc function \type\()_bilin_16bpc_neon, export=1 .ifc \bdmax, w8 ldr w8, [sp] .endif dup v1.8h, \mx dup v3.8h, \my mov w10, #16 sub w9, w10, \mx sub w10, w10, \my dup v0.8h, w9 dup v2.8h, w10 .ifc \type, prep uxtw \d_strd, \w lsl \d_strd, \d_strd, #1 .endif clz \bdmax, \bdmax // bitdepth_max clz w9, \w sub \bdmax, \bdmax, #18 // intermediate_bits = clz(bitdepth_max) - 18 mov w11, #4 sub w9, w9, #24 sub w11, w11, \bdmax // 4 - intermediate_bits add w12, \bdmax, #4 // 4 + intermediate_bits cbnz \mx, L(\type\()_bilin_h) cbnz \my, L(\type\()_bilin_v) b \type\()_neon L(\type\()_bilin_h): cbnz \my, L(\type\()_bilin_hv) adr x10, L(\type\()_bilin_h_tbl) dup v31.8h, w11 // 4 - intermediate_bits ldrh w9, [x10, x9, lsl #1] neg v31.8h, v31.8h // -(4-intermediate_bits) .ifc \type, put dup v30.8h, \bdmax // intermediate_bits .else movi v29.8h, #(PREP_BIAS >> 8), lsl #8 .endif sub x10, x10, w9, uxtw .ifc \type, put neg v30.8h, v30.8h // -intermediate_bits .endif br x10 20: // 2xN h AARCH64_VALID_JUMP_TARGET .ifc \type, put add \ds2, \dst, \d_strd add \sr2, \src, \s_strd lsl \d_strd, \d_strd, #1 lsl \s_strd, \s_strd, #1 2: ld1 {v4.4h}, [\src], \s_strd ld1 {v6.4h}, [\sr2], \s_strd ext v5.8b, v4.8b, v4.8b, #2 ext v7.8b, v6.8b, v6.8b, #2 trn1 v4.2s, v4.2s, v6.2s trn1 v5.2s, v5.2s, v7.2s subs \h, \h, #2 mul v4.4h, v4.4h, v0.4h mla v4.4h, v5.4h, v1.4h urshl v4.4h, v4.4h, v31.4h urshl v4.4h, v4.4h, v30.4h st1 {v4.s}[0], [\dst], \d_strd st1 {v4.s}[1], [\ds2], \d_strd b.gt 2b ret .endif 40: // 4xN h AARCH64_VALID_JUMP_TARGET add \ds2, \dst, \d_strd add \sr2, \src, \s_strd lsl \d_strd, \d_strd, #1 lsl \s_strd, \s_strd, #1 4: ld1 {v4.8h}, [\src], \s_strd ld1 {v6.8h}, [\sr2], \s_strd ext v5.16b, v4.16b, v4.16b, #2 ext v7.16b, v6.16b, v6.16b, #2 trn1 v4.2d, v4.2d, v6.2d trn1 v5.2d, v5.2d, v7.2d subs \h, \h, #2 mul v4.8h, v4.8h, v0.8h mla v4.8h, v5.8h, v1.8h urshl v4.8h, v4.8h, v31.8h .ifc \type, put urshl v4.8h, v4.8h, v30.8h .else sub v4.8h, v4.8h, v29.8h .endif st1 {v4.d}[0], [\dst], \d_strd st1 {v4.d}[1], [\ds2], \d_strd b.gt 4b ret 80: // 8xN h AARCH64_VALID_JUMP_TARGET add \ds2, \dst, \d_strd add \sr2, \src, \s_strd lsl \d_strd, \d_strd, #1 lsl \s_strd, \s_strd, #1 8: ldr h5, [\src, #16] ldr h7, [\sr2, #16] ld1 {v4.8h}, [\src], \s_strd ld1 {v6.8h}, [\sr2], \s_strd ext v5.16b, v4.16b, v5.16b, #2 ext v7.16b, v6.16b, v7.16b, #2 subs \h, \h, #2 mul v4.8h, v4.8h, v0.8h mla v4.8h, v5.8h, v1.8h mul v6.8h, v6.8h, v0.8h mla v6.8h, v7.8h, v1.8h urshl v4.8h, v4.8h, v31.8h urshl v6.8h, v6.8h, v31.8h .ifc \type, put urshl v4.8h, v4.8h, v30.8h urshl v6.8h, v6.8h, v30.8h .else sub v4.8h, v4.8h, v29.8h sub v6.8h, v6.8h, v29.8h .endif st1 {v4.8h}, [\dst], \d_strd st1 {v6.8h}, [\ds2], \d_strd b.gt 8b ret 160: 320: 640: 1280: // 16xN, 32xN, ... h AARCH64_VALID_JUMP_TARGET add \ds2, \dst, \d_strd add \sr2, \src, \s_strd lsl \s_strd, \s_strd, #1 sub \s_strd, \s_strd, \w, uxtw #1 sub \s_strd, \s_strd, #16 .ifc \type, put lsl \d_strd, \d_strd, #1 sub \d_strd, \d_strd, \w, uxtw #1 .endif 161: ld1 {v16.8h}, [\src], #16 ld1 {v21.8h}, [\sr2], #16 mov \mx, \w 16: ld1 {v17.8h, v18.8h}, [\src], #32 ld1 {v22.8h, v23.8h}, [\sr2], #32 ext v19.16b, v16.16b, v17.16b, #2 ext v20.16b, v17.16b, v18.16b, #2 ext v24.16b, v21.16b, v22.16b, #2 ext v25.16b, v22.16b, v23.16b, #2 mul v16.8h, v16.8h, v0.8h mla v16.8h, v19.8h, v1.8h mul v17.8h, v17.8h, v0.8h mla v17.8h, v20.8h, v1.8h mul v21.8h, v21.8h, v0.8h mla v21.8h, v24.8h, v1.8h mul v22.8h, v22.8h, v0.8h mla v22.8h, v25.8h, v1.8h urshl v16.8h, v16.8h, v31.8h urshl v17.8h, v17.8h, v31.8h urshl v21.8h, v21.8h, v31.8h urshl v22.8h, v22.8h, v31.8h subs \mx, \mx, #16 .ifc \type, put urshl v16.8h, v16.8h, v30.8h urshl v17.8h, v17.8h, v30.8h urshl v21.8h, v21.8h, v30.8h urshl v22.8h, v22.8h, v30.8h .else sub v16.8h, v16.8h, v29.8h sub v17.8h, v17.8h, v29.8h sub v21.8h, v21.8h, v29.8h sub v22.8h, v22.8h, v29.8h .endif st1 {v16.8h, v17.8h}, [\dst], #32 st1 {v21.8h, v22.8h}, [\ds2], #32 b.le 9f mov v16.16b, v18.16b mov v21.16b, v23.16b b 16b 9: add \dst, \dst, \d_strd add \ds2, \ds2, \d_strd add \src, \src, \s_strd add \sr2, \sr2, \s_strd subs \h, \h, #2 b.gt 161b ret L(\type\()_bilin_h_tbl): .hword L(\type\()_bilin_h_tbl) - 1280b .hword L(\type\()_bilin_h_tbl) - 640b .hword L(\type\()_bilin_h_tbl) - 320b .hword L(\type\()_bilin_h_tbl) - 160b .hword L(\type\()_bilin_h_tbl) - 80b .hword L(\type\()_bilin_h_tbl) - 40b .hword L(\type\()_bilin_h_tbl) - 20b .hword 0 L(\type\()_bilin_v): cmp \h, #4 adr x10, L(\type\()_bilin_v_tbl) .ifc \type, prep dup v31.8h, w11 // 4 - intermediate_bits .endif ldrh w9, [x10, x9, lsl #1] .ifc \type, prep movi v29.8h, #(PREP_BIAS >> 8), lsl #8 neg v31.8h, v31.8h // -(4-intermediate_bits) .endif sub x10, x10, w9, uxtw br x10 20: // 2xN v AARCH64_VALID_JUMP_TARGET .ifc \type, put cmp \h, #2 add \ds2, \dst, \d_strd add \sr2, \src, \s_strd lsl \s_strd, \s_strd, #1 lsl \d_strd, \d_strd, #1 // 2x2 v ld1 {v16.s}[0], [\src], \s_strd b.gt 24f 22: ld1 {v17.s}[0], [\sr2], \s_strd ld1 {v18.s}[0], [\src], \s_strd trn1 v16.2s, v16.2s, v17.2s trn1 v17.2s, v17.2s, v18.2s mul v4.4h, v16.4h, v2.4h mla v4.4h, v17.4h, v3.4h urshr v4.8h, v4.8h, #4 st1 {v4.s}[0], [\dst] st1 {v4.s}[1], [\ds2] ret 24: // 2x4, 2x6, 2x8, ... v ld1 {v17.s}[0], [\sr2], \s_strd ld1 {v18.s}[0], [\src], \s_strd ld1 {v19.s}[0], [\sr2], \s_strd ld1 {v20.s}[0], [\src], \s_strd sub \h, \h, #4 trn1 v16.2s, v16.2s, v17.2s trn1 v17.2s, v17.2s, v18.2s trn1 v18.2s, v18.2s, v19.2s trn1 v19.2s, v19.2s, v20.2s trn1 v16.2d, v16.2d, v18.2d trn1 v17.2d, v17.2d, v19.2d mul v4.8h, v16.8h, v2.8h mla v4.8h, v17.8h, v3.8h cmp \h, #2 urshr v4.8h, v4.8h, #4 st1 {v4.s}[0], [\dst], \d_strd st1 {v4.s}[1], [\ds2], \d_strd st1 {v4.s}[2], [\dst], \d_strd st1 {v4.s}[3], [\ds2], \d_strd b.lt 0f mov v16.8b, v20.8b b.eq 22b b 24b 0: ret .endif 40: // 4xN v AARCH64_VALID_JUMP_TARGET add \ds2, \dst, \d_strd add \sr2, \src, \s_strd lsl \s_strd, \s_strd, #1 lsl \d_strd, \d_strd, #1 ld1 {v16.4h}, [\src], \s_strd 4: ld1 {v17.4h}, [\sr2], \s_strd ld1 {v18.4h}, [\src], \s_strd trn1 v16.2d, v16.2d, v17.2d trn1 v17.2d, v17.2d, v18.2d mul v4.8h, v16.8h, v2.8h mla v4.8h, v17.8h, v3.8h subs \h, \h, #2 .ifc \type, put urshr v4.8h, v4.8h, #4 .else urshl v4.8h, v4.8h, v31.8h sub v4.8h, v4.8h, v29.8h .endif st1 {v4.d}[0], [\dst], \d_strd st1 {v4.d}[1], [\ds2], \d_strd b.le 0f mov v16.8b, v18.8b b 4b 0: ret 80: // 8xN v AARCH64_VALID_JUMP_TARGET add \ds2, \dst, \d_strd add \sr2, \src, \s_strd lsl \s_strd, \s_strd, #1 lsl \d_strd, \d_strd, #1 ld1 {v16.8h}, [\src], \s_strd 8: ld1 {v17.8h}, [\sr2], \s_strd ld1 {v18.8h}, [\src], \s_strd mul v4.8h, v16.8h, v2.8h mla v4.8h, v17.8h, v3.8h mul v5.8h, v17.8h, v2.8h mla v5.8h, v18.8h, v3.8h subs \h, \h, #2 .ifc \type, put urshr v4.8h, v4.8h, #4 urshr v5.8h, v5.8h, #4 .else urshl v4.8h, v4.8h, v31.8h urshl v5.8h, v5.8h, v31.8h sub v4.8h, v4.8h, v29.8h sub v5.8h, v5.8h, v29.8h .endif st1 {v4.8h}, [\dst], \d_strd st1 {v5.8h}, [\ds2], \d_strd b.le 0f mov v16.16b, v18.16b b 8b 0: ret 160: // 16xN, 32xN, ... 320: 640: 1280: AARCH64_VALID_JUMP_TARGET mov \my, \h 1: add \ds2, \dst, \d_strd add \sr2, \src, \s_strd lsl \s_strd, \s_strd, #1 lsl \d_strd, \d_strd, #1 ld1 {v16.8h, v17.8h}, [\src], \s_strd 2: ld1 {v18.8h, v19.8h}, [\sr2], \s_strd ld1 {v20.8h, v21.8h}, [\src], \s_strd mul v4.8h, v16.8h, v2.8h mla v4.8h, v18.8h, v3.8h mul v5.8h, v17.8h, v2.8h mla v5.8h, v19.8h, v3.8h mul v6.8h, v18.8h, v2.8h mla v6.8h, v20.8h, v3.8h mul v7.8h, v19.8h, v2.8h mla v7.8h, v21.8h, v3.8h subs \h, \h, #2 .ifc \type, put urshr v4.8h, v4.8h, #4 urshr v5.8h, v5.8h, #4 urshr v6.8h, v6.8h, #4 urshr v7.8h, v7.8h, #4 .else urshl v4.8h, v4.8h, v31.8h urshl v5.8h, v5.8h, v31.8h urshl v6.8h, v6.8h, v31.8h urshl v7.8h, v7.8h, v31.8h sub v4.8h, v4.8h, v29.8h sub v5.8h, v5.8h, v29.8h sub v6.8h, v6.8h, v29.8h sub v7.8h, v7.8h, v29.8h .endif st1 {v4.8h, v5.8h}, [\dst], \d_strd st1 {v6.8h, v7.8h}, [\ds2], \d_strd b.le 9f mov v16.16b, v20.16b mov v17.16b, v21.16b b 2b 9: subs \w, \w, #16 b.le 0f asr \s_strd, \s_strd, #1 asr \d_strd, \d_strd, #1 msub \src, \s_strd, \xmy, \src msub \dst, \d_strd, \xmy, \dst sub \src, \src, \s_strd, lsl #1 mov \h, \my add \src, \src, #32 add \dst, \dst, #32 b 1b 0: ret L(\type\()_bilin_v_tbl): .hword L(\type\()_bilin_v_tbl) - 1280b .hword L(\type\()_bilin_v_tbl) - 640b .hword L(\type\()_bilin_v_tbl) - 320b .hword L(\type\()_bilin_v_tbl) - 160b .hword L(\type\()_bilin_v_tbl) - 80b .hword L(\type\()_bilin_v_tbl) - 40b .hword L(\type\()_bilin_v_tbl) - 20b .hword 0 L(\type\()_bilin_hv): adr x10, L(\type\()_bilin_hv_tbl) dup v31.8h, w11 // 4 - intermediate_bits ldrh w9, [x10, x9, lsl #1] neg v31.8h, v31.8h // -(4-intermediate_bits) .ifc \type, put dup v30.4s, w12 // 4 + intermediate_bits .else movi v29.8h, #(PREP_BIAS >> 8), lsl #8 .endif sub x10, x10, w9, uxtw .ifc \type, put neg v30.4s, v30.4s // -(4+intermediate_bits) .endif br x10 20: // 2xN hv AARCH64_VALID_JUMP_TARGET .ifc \type, put add \sr2, \src, \s_strd add \ds2, \dst, \d_strd lsl \s_strd, \s_strd, #1 lsl \d_strd, \d_strd, #1 ld1 {v20.4h}, [\src], \s_strd ext v21.8b, v20.8b, v20.8b, #2 mul v16.4h, v20.4h, v0.4h mla v16.4h, v21.4h, v1.4h urshl v16.4h, v16.4h, v31.4h 2: ld1 {v22.4h}, [\sr2], \s_strd ld1 {v24.4h}, [\src], \s_strd ext v23.8b, v22.8b, v22.8b, #2 ext v25.8b, v24.8b, v24.8b, #2 trn1 v22.2s, v22.2s, v24.2s trn1 v23.2s, v23.2s, v25.2s mul v17.4h, v22.4h, v0.4h mla v17.4h, v23.4h, v1.4h urshl v17.4h, v17.4h, v31.4h trn1 v16.2s, v16.2s, v17.2s umull v4.4s, v16.4h, v2.4h umlal v4.4s, v17.4h, v3.4h urshl v4.4s, v4.4s, v30.4s xtn v4.4h, v4.4s subs \h, \h, #2 st1 {v4.s}[0], [\dst], \d_strd st1 {v4.s}[1], [\ds2], \d_strd b.le 0f trn2 v16.2s, v17.2s, v17.2s b 2b 0: ret .endif 40: // 4xN hv AARCH64_VALID_JUMP_TARGET add \sr2, \src, \s_strd add \ds2, \dst, \d_strd lsl \s_strd, \s_strd, #1 lsl \d_strd, \d_strd, #1 ld1 {v20.8h}, [\src], \s_strd ext v21.16b, v20.16b, v20.16b, #2 mul v16.4h, v20.4h, v0.4h mla v16.4h, v21.4h, v1.4h urshl v16.4h, v16.4h, v31.4h 4: ld1 {v22.8h}, [\sr2], \s_strd ld1 {v24.8h}, [\src], \s_strd ext v23.16b, v22.16b, v22.16b, #2 ext v25.16b, v24.16b, v24.16b, #2 trn1 v22.2d, v22.2d, v24.2d trn1 v23.2d, v23.2d, v25.2d mul v17.8h, v22.8h, v0.8h mla v17.8h, v23.8h, v1.8h urshl v17.8h, v17.8h, v31.8h trn1 v16.2d, v16.2d, v17.2d umull v4.4s, v16.4h, v2.4h umlal v4.4s, v17.4h, v3.4h umull2 v5.4s, v16.8h, v2.8h umlal2 v5.4s, v17.8h, v3.8h .ifc \type, put urshl v4.4s, v4.4s, v30.4s urshl v5.4s, v5.4s, v30.4s uzp1 v4.8h, v4.8h, v5.8h // Same as xtn, xtn2 .else rshrn v4.4h, v4.4s, #4 rshrn2 v4.8h, v5.4s, #4 sub v4.8h, v4.8h, v29.8h .endif subs \h, \h, #2 st1 {v4.d}[0], [\dst], \d_strd st1 {v4.d}[1], [\ds2], \d_strd b.le 0f trn2 v16.2d, v17.2d, v17.2d b 4b 0: ret 80: // 8xN, 16xN, ... hv 160: 320: 640: 1280: AARCH64_VALID_JUMP_TARGET mov \my, \h 1: add \sr2, \src, \s_strd add \ds2, \dst, \d_strd lsl \s_strd, \s_strd, #1 lsl \d_strd, \d_strd, #1 ldr h21, [\src, #16] ld1 {v20.8h}, [\src], \s_strd ext v21.16b, v20.16b, v21.16b, #2 mul v16.8h, v20.8h, v0.8h mla v16.8h, v21.8h, v1.8h urshl v16.8h, v16.8h, v31.8h 2: ldr h23, [\sr2, #16] ld1 {v22.8h}, [\sr2], \s_strd ldr h25, [\src, #16] ld1 {v24.8h}, [\src], \s_strd ext v23.16b, v22.16b, v23.16b, #2 ext v25.16b, v24.16b, v25.16b, #2 mul v17.8h, v22.8h, v0.8h mla v17.8h, v23.8h, v1.8h mul v18.8h, v24.8h, v0.8h mla v18.8h, v25.8h, v1.8h urshl v17.8h, v17.8h, v31.8h urshl v18.8h, v18.8h, v31.8h umull v4.4s, v16.4h, v2.4h umlal v4.4s, v17.4h, v3.4h umull2 v5.4s, v16.8h, v2.8h umlal2 v5.4s, v17.8h, v3.8h umull v6.4s, v17.4h, v2.4h umlal v6.4s, v18.4h, v3.4h umull2 v7.4s, v17.8h, v2.8h umlal2 v7.4s, v18.8h, v3.8h .ifc \type, put urshl v4.4s, v4.4s, v30.4s urshl v5.4s, v5.4s, v30.4s urshl v6.4s, v6.4s, v30.4s urshl v7.4s, v7.4s, v30.4s uzp1 v4.8h, v4.8h, v5.8h // Same as xtn, xtn2 uzp1 v5.8h, v6.8h, v7.8h // Ditto .else rshrn v4.4h, v4.4s, #4 rshrn2 v4.8h, v5.4s, #4 rshrn v5.4h, v6.4s, #4 rshrn2 v5.8h, v7.4s, #4 sub v4.8h, v4.8h, v29.8h sub v5.8h, v5.8h, v29.8h .endif subs \h, \h, #2 st1 {v4.8h}, [\dst], \d_strd st1 {v5.8h}, [\ds2], \d_strd b.le 9f mov v16.16b, v18.16b b 2b 9: subs \w, \w, #8 b.le 0f asr \s_strd, \s_strd, #1 asr \d_strd, \d_strd, #1 msub \src, \s_strd, \xmy, \src msub \dst, \d_strd, \xmy, \dst sub \src, \src, \s_strd, lsl #1 mov \h, \my add \src, \src, #16 add \dst, \dst, #16 b 1b 0: ret L(\type\()_bilin_hv_tbl): .hword L(\type\()_bilin_hv_tbl) - 1280b .hword L(\type\()_bilin_hv_tbl) - 640b .hword L(\type\()_bilin_hv_tbl) - 320b .hword L(\type\()_bilin_hv_tbl) - 160b .hword L(\type\()_bilin_hv_tbl) - 80b .hword L(\type\()_bilin_hv_tbl) - 40b .hword L(\type\()_bilin_hv_tbl) - 20b .hword 0 endfunc .endm filter_fn put, x0, x1, x2, x3, w4, w5, w6, x6, w7, x7, w8, x9, x10 filter_fn prep, x0, x8, x1, x2, w3, w4, w5, x5, w6, x6, w7, x9, x10 .macro load_filter_row dst, src, inc asr w13, \src, #10 add \src, \src, \inc ldr \dst, [x11, w13, sxtw #3] .endm function warp_filter_horz_neon add w12, w5, #512 ld1 {v16.8h, v17.8h}, [x2], x3 load_filter_row d0, w12, w7 load_filter_row d1, w12, w7 load_filter_row d2, w12, w7 sxtl v0.8h, v0.8b load_filter_row d3, w12, w7 sxtl v1.8h, v1.8b load_filter_row d4, w12, w7 sxtl v2.8h, v2.8b load_filter_row d5, w12, w7 sxtl v3.8h, v3.8b load_filter_row d6, w12, w7 sxtl v4.8h, v4.8b load_filter_row d7, w12, w7 sxtl v5.8h, v5.8b ext v18.16b, v16.16b, v17.16b, #2*1 smull v8.4s, v16.4h, v0.4h smull2 v9.4s, v16.8h, v0.8h sxtl v6.8h, v6.8b ext v19.16b, v16.16b, v17.16b, #2*2 smull v10.4s, v18.4h, v1.4h smull2 v11.4s, v18.8h, v1.8h sxtl v7.8h, v7.8b ext v20.16b, v16.16b, v17.16b, #2*3 smull v0.4s, v19.4h, v2.4h smull2 v1.4s, v19.8h, v2.8h ext v21.16b, v16.16b, v17.16b, #2*4 addp v8.4s, v8.4s, v9.4s smull v2.4s, v20.4h, v3.4h smull2 v3.4s, v20.8h, v3.8h ext v22.16b, v16.16b, v17.16b, #2*5 addp v9.4s, v10.4s, v11.4s smull v10.4s, v21.4h, v4.4h smull2 v11.4s, v21.8h, v4.8h ext v23.16b, v16.16b, v17.16b, #2*6 addp v0.4s, v0.4s, v1.4s smull v18.4s, v22.4h, v5.4h smull2 v19.4s, v22.8h, v5.8h ext v16.16b, v16.16b, v17.16b, #2*7 addp v1.4s, v2.4s, v3.4s addp v2.4s, v10.4s, v11.4s smull v20.4s, v23.4h, v6.4h smull2 v21.4s, v23.8h, v6.8h addp v3.4s, v18.4s, v19.4s smull v22.4s, v16.4h, v7.4h smull2 v23.4s, v16.8h, v7.8h addp v4.4s, v20.4s, v21.4s addp v5.4s, v22.4s, v23.4s addp v8.4s, v8.4s, v9.4s addp v0.4s, v0.4s, v1.4s addp v2.4s, v2.4s, v3.4s addp v4.4s, v4.4s, v5.4s addp v16.4s, v8.4s, v0.4s addp v17.4s, v2.4s, v4.4s add w5, w5, w8 srshl v16.4s, v16.4s, v14.4s // -(7 - intermediate_bits) srshl v17.4s, v17.4s, v14.4s // -(7 - intermediate_bits) ret endfunc // void dav1d_warp_affine_8x8_16bpc_neon( // pixel *dst, const ptrdiff_t dst_stride, // const pixel *src, const ptrdiff_t src_stride, // const int16_t *const abcd, int mx, int my, // const int bitdepth_max) .macro warp t function warp_affine_8x8\t\()_16bpc_neon, export=1 stp d8, d9, [sp, #-0x40]! stp d10, d11, [sp, #0x10] stp d12, d13, [sp, #0x20] stp d14, d15, [sp, #0x30] .ifb \t dup v15.8h, w7 // bitdepth_max .else movi v15.8h, #(PREP_BIAS >> 8), lsl #8 .endif clz w7, w7 // intermediate_bits = clz(bitdepth_max) - 18 .ifb \t sub w8, w7, #11 // 7 + intermediate_bits = clz(bitdepth_max) - 18 + 7 .endif sub w7, w7, #25 // -(7 - intermediate_bits) .ifb \t neg w8, w8 // -(7 + intermediate_bits) .endif dup v14.4s, w7 // -(7 - intermediate_bits) .ifb \t dup v13.4s, w8 // -(7 + intermediate_bits) .endif ldr x4, [x4] sbfx x7, x4, #0, #16 sbfx x8, x4, #16, #16 sbfx x9, x4, #32, #16 sbfx x4, x4, #48, #16 mov w10, #8 sub x2, x2, x3, lsl #1 sub x2, x2, x3 sub x2, x2, #6 movrel x11, X(mc_warp_filter), 64*8 mov x15, x30 .ifnb \t lsl x1, x1, #1 .endif bl warp_filter_horz_neon uzp1 v24.8h, v16.8h, v17.8h // Same as xtn, xtn2 bl warp_filter_horz_neon uzp1 v25.8h, v16.8h, v17.8h // Ditto bl warp_filter_horz_neon uzp1 v26.8h, v16.8h, v17.8h // Ditto bl warp_filter_horz_neon uzp1 v27.8h, v16.8h, v17.8h // Ditto bl warp_filter_horz_neon uzp1 v28.8h, v16.8h, v17.8h // Ditto bl warp_filter_horz_neon uzp1 v29.8h, v16.8h, v17.8h // Ditto bl warp_filter_horz_neon uzp1 v30.8h, v16.8h, v17.8h // Ditto 1: add w14, w6, #512 bl warp_filter_horz_neon uzp1 v31.8h, v16.8h, v17.8h // Same as xtn, xtn2 load_filter_row d0, w14, w9 load_filter_row d1, w14, w9 load_filter_row d2, w14, w9 load_filter_row d3, w14, w9 load_filter_row d4, w14, w9 load_filter_row d5, w14, w9 load_filter_row d6, w14, w9 load_filter_row d7, w14, w9 transpose_8x8b_xtl v0, v1, v2, v3, v4, v5, v6, v7, sxtl // This ordering of smull/smlal/smull2/smlal2 is highly // beneficial for Cortex A53 here. smull v16.4s, v24.4h, v0.4h smlal v16.4s, v25.4h, v1.4h smlal v16.4s, v26.4h, v2.4h smlal v16.4s, v27.4h, v3.4h smlal v16.4s, v28.4h, v4.4h smlal v16.4s, v29.4h, v5.4h smlal v16.4s, v30.4h, v6.4h smlal v16.4s, v31.4h, v7.4h smull2 v17.4s, v24.8h, v0.8h smlal2 v17.4s, v25.8h, v1.8h smlal2 v17.4s, v26.8h, v2.8h smlal2 v17.4s, v27.8h, v3.8h smlal2 v17.4s, v28.8h, v4.8h smlal2 v17.4s, v29.8h, v5.8h smlal2 v17.4s, v30.8h, v6.8h smlal2 v17.4s, v31.8h, v7.8h mov v24.16b, v25.16b mov v25.16b, v26.16b .ifb \t srshl v16.4s, v16.4s, v13.4s // -(7 + intermediate_bits) srshl v17.4s, v17.4s, v13.4s // -(7 + intermediate_bits) .else rshrn v16.4h, v16.4s, #7 rshrn2 v16.8h, v17.4s, #7 .endif mov v26.16b, v27.16b .ifb \t sqxtun v16.4h, v16.4s sqxtun2 v16.8h, v17.4s .else sub v16.8h, v16.8h, v15.8h // PREP_BIAS .endif mov v27.16b, v28.16b mov v28.16b, v29.16b .ifb \t umin v16.8h, v16.8h, v15.8h // bitdepth_max .endif mov v29.16b, v30.16b mov v30.16b, v31.16b subs w10, w10, #1 st1 {v16.8h}, [x0], x1 add w6, w6, w4 b.gt 1b ldp d14, d15, [sp, #0x30] ldp d12, d13, [sp, #0x20] ldp d10, d11, [sp, #0x10] ldp d8, d9, [sp], 0x40 ret x15 endfunc .endm warp warp t // void dav1d_emu_edge_16bpc_neon( // const intptr_t bw, const intptr_t bh, // const intptr_t iw, const intptr_t ih, // const intptr_t x, const intptr_t y, // pixel *dst, const ptrdiff_t dst_stride, // const pixel *ref, const ptrdiff_t ref_stride) function emu_edge_16bpc_neon, export=1 ldp x8, x9, [sp] // ref += iclip(y, 0, ih - 1) * PXSTRIDE(ref_stride) // ref += iclip(x, 0, iw - 1) sub x12, x3, #1 // ih - 1 cmp x5, x3 sub x13, x2, #1 // iw - 1 csel x12, x12, x5, ge // min(y, ih - 1) cmp x4, x2 bic x12, x12, x12, asr #63 // max(min(y, ih - 1), 0) csel x13, x13, x4, ge // min(x, iw - 1) bic x13, x13, x13, asr #63 // max(min(x, iw - 1), 0) madd x8, x12, x9, x8 // ref += iclip() * stride add x8, x8, x13, lsl #1 // ref += iclip() // bottom_ext = iclip(y + bh - ih, 0, bh - 1) // top_ext = iclip(-y, 0, bh - 1) add x10, x5, x1 // y + bh neg x5, x5 // -y sub x10, x10, x3 // y + bh - ih sub x12, x1, #1 // bh - 1 cmp x10, x1 bic x5, x5, x5, asr #63 // max(-y, 0) csel x10, x10, x12, lt // min(y + bh - ih, bh-1) cmp x5, x1 bic x10, x10, x10, asr #63 // max(min(y + bh - ih, bh-1), 0) csel x5, x5, x12, lt // min(max(-y, 0), bh-1) // right_ext = iclip(x + bw - iw, 0, bw - 1) // left_ext = iclip(-x, 0, bw - 1) add x11, x4, x0 // x + bw neg x4, x4 // -x sub x11, x11, x2 // x + bw - iw sub x13, x0, #1 // bw - 1 cmp x11, x0 bic x4, x4, x4, asr #63 // max(-x, 0) csel x11, x11, x13, lt // min(x + bw - iw, bw-1) cmp x4, x0 bic x11, x11, x11, asr #63 // max(min(x + bw - iw, bw-1), 0) csel x4, x4, x13, lt // min(max(-x, 0), bw - 1) // center_h = bh - top_ext - bottom_ext // dst += top_ext * PXSTRIDE(dst_stride) // center_w = bw - left_ext - right_ext sub x1, x1, x5 // bh - top_ext madd x6, x5, x7, x6 sub x2, x0, x4 // bw - left_ext sub x1, x1, x10 // center_h = bh - top_ext - bottom_ext sub x2, x2, x11 // center_w = bw - left_ext - right_ext mov x14, x6 // backup of dst .macro v_loop need_left, need_right 0: .if \need_left ld1r {v0.8h}, [x8] mov x12, x6 // out = dst mov x3, x4 mov v1.16b, v0.16b 1: subs x3, x3, #16 st1 {v0.8h, v1.8h}, [x12], #32 b.gt 1b .endif mov x13, x8 add x12, x6, x4, lsl #1 // out = dst + left_ext mov x3, x2 1: ld1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x13], #64 subs x3, x3, #32 st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x12], #64 b.gt 1b .if \need_right add x3, x8, x2, lsl #1 // in + center_w sub x3, x3, #2 // in + center_w - 1 add x12, x6, x4, lsl #1 // dst + left_ext ld1r {v0.8h}, [x3] add x12, x12, x2, lsl #1 // out = dst + left_ext + center_w mov x3, x11 mov v1.16b, v0.16b 1: subs x3, x3, #16 st1 {v0.8h, v1.8h}, [x12], #32 b.gt 1b .endif subs x1, x1, #1 // center_h-- add x6, x6, x7 add x8, x8, x9 b.gt 0b .endm cbz x4, 2f // need_left cbz x11, 3f // need_left + need_right v_loop 1, 1 b 5f 2: // !need_left cbz x11, 4f // !need_left + need_right v_loop 0, 1 b 5f 3: // need_left + !need_right v_loop 1, 0 b 5f 4: // !need_left + !need_right v_loop 0, 0 5: cbz x10, 3f // need_bottom sub x8, x6, x7 // ref = dst - stride mov x4, x0 1: ld1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x8], #64 mov x3, x10 2: subs x3, x3, #1 st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x6], x7 b.gt 2b msub x6, x7, x10, x6 // dst -= bottom_ext * stride subs x4, x4, #32 // bw -= 32 add x6, x6, #64 // dst += 32 b.gt 1b 3: cbz x5, 3f // need_top msub x6, x7, x5, x14 // dst = stored_dst - top_ext * stride 1: ld1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x14], #64 mov x3, x5 2: subs x3, x3, #1 st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x6], x7 b.gt 2b msub x6, x7, x5, x6 // dst -= top_ext * stride subs x0, x0, #32 // bw -= 32 add x6, x6, #64 // dst += 32 b.gt 1b 3: ret endfunc