/* * Copyright © 2018, VideoLAN and dav1d authors * Copyright © 2018, Janne Grunau * Copyright © 2018, Martin Storsjo * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, this * list of conditions and the following disclaimer. * * 2. Redistributions in binary form must reproduce the above copyright notice, * this list of conditions and the following disclaimer in the documentation * and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include "src/arm/asm.S" #include "util.S" .macro avg dst0, dst1, t0, t1, t2, t3 vld1.16 {\t0,\t1}, [r2, :128]! vld1.16 {\t2,\t3}, [r3, :128]! vadd.i16 \t0, \t0, \t2 vadd.i16 \t1, \t1, \t3 vqrshrun.s16 \dst0, \t0, #5 vqrshrun.s16 \dst1, \t1, #5 .endm .macro w_avg dst0, dst1, t0, t1, t2, t3 vld1.16 {\t0,\t1}, [r2, :128]! vld1.16 {\t2,\t3}, [r3, :128]! vsub.i16 \t0, \t2, \t0 vsub.i16 \t1, \t3, \t1 vqdmulh.s16 \t0, \t0, q15 vqdmulh.s16 \t1, \t1, q15 vadd.i16 \t0, \t2, \t0 vadd.i16 \t1, \t3, \t1 vqrshrun.s16 \dst0, \t0, #4 vqrshrun.s16 \dst1, \t1, #4 .endm .macro mask dst0, dst1, t0, t1, t2, t3 vld1.8 {q14}, [lr, :128]! vld1.16 {\t0,\t1}, [r2, :128]! vmul.i8 q14, q14, q15 vld1.16 {\t2,\t3}, [r3, :128]! vshll.i8 q13, d28, #8 vshll.i8 q14, d29, #8 vsub.i16 \t0, \t2, \t0 vsub.i16 \t1, \t3, \t1 vqdmulh.s16 \t0, \t0, q13 vqdmulh.s16 \t1, \t1, q14 vadd.i16 \t0, \t2, \t0 vadd.i16 \t1, \t3, \t1 vqrshrun.s16 \dst0, \t0, #4 vqrshrun.s16 \dst1, \t1, #4 .endm .macro bidir_fn type function \type\()_8bpc_neon, export=1 push {r4-r6,lr} ldrd r4, r5, [sp, #16] clz r4, r4 .ifnc \type, avg ldr lr, [sp, #24] .endif .ifc \type, w_avg vdup.s16 q15, lr vneg.s16 q15, q15 vshl.i16 q15, q15, #11 .endif .ifc \type, mask vmov.i8 q15, #256-2 .endif adr r12, L(\type\()_tbl) sub r4, r4, #24 ldr r4, [r12, r4, lsl #2] \type d16, d17, q0, q1, q2, q3 add r12, r12, r4 bx r12 .align 2 L(\type\()_tbl): .word 1280f - L(\type\()_tbl) + CONFIG_THUMB .word 640f - L(\type\()_tbl) + CONFIG_THUMB .word 320f - L(\type\()_tbl) + CONFIG_THUMB .word 160f - L(\type\()_tbl) + CONFIG_THUMB .word 80f - L(\type\()_tbl) + CONFIG_THUMB .word 4f - L(\type\()_tbl) + CONFIG_THUMB 4: add r6, r0, r1 lsl r1, r1, #1 cmp r5, #4 vst1.32 {d16[0]}, [r0, :32], r1 vst1.32 {d16[1]}, [r6, :32], r1 vst1.32 {d17[0]}, [r0, :32], r1 vst1.32 {d17[1]}, [r6, :32], r1 beq 0f \type d18, d19, q0, q1, q2, q3 cmp r5, #8 vst1.32 {d18[0]}, [r0, :32], r1 vst1.32 {d18[1]}, [r6, :32], r1 vst1.32 {d19[0]}, [r0, :32], r1 vst1.32 {d19[1]}, [r6, :32], r1 beq 0f \type d16, d17, q0, q1, q2, q3 vst1.32 {d16[0]}, [r0, :32], r1 vst1.32 {d16[1]}, [r6, :32], r1 \type d18, d19, q0, q1, q2, q3 vst1.32 {d17[0]}, [r0, :32], r1 vst1.32 {d17[1]}, [r6, :32], r1 vst1.32 {d18[0]}, [r0, :32], r1 vst1.32 {d18[1]}, [r6, :32], r1 vst1.32 {d19[0]}, [r0, :32], r1 vst1.32 {d19[1]}, [r6, :32], r1 pop {r4-r6,pc} 80: add r6, r0, r1 lsl r1, r1, #1 8: vst1.8 {d16}, [r0, :64], r1 \type d18, d19, q0, q1, q2, q3 vst1.8 {d17}, [r6, :64], r1 vst1.8 {d18}, [r0, :64], r1 subs r5, r5, #4 vst1.8 {d19}, [r6, :64], r1 ble 0f \type d16, d17, q0, q1, q2, q3 b 8b 160: add r6, r0, r1 lsl r1, r1, #1 16: \type d18, d19, q0, q1, q2, q3 vst1.8 {q8}, [r0, :128], r1 \type d20, d21, q0, q1, q2, q3 vst1.8 {q9}, [r6, :128], r1 \type d22, d23, q0, q1, q2, q3 vst1.8 {q10}, [r0, :128], r1 subs r5, r5, #4 vst1.8 {q11}, [r6, :128], r1 ble 0f \type d16, d17, q0, q1, q2, q3 b 16b 320: add r6, r0, r1 lsl r1, r1, #1 32: \type d18, d19, q0, q1, q2, q3 \type d20, d21, q0, q1, q2, q3 vst1.8 {q8, q9}, [r0, :128], r1 \type d22, d23, q0, q1, q2, q3 subs r5, r5, #2 vst1.8 {q10, q11}, [r6, :128], r1 ble 0f \type d16, d17, q0, q1, q2, q3 b 32b 640: add r6, r0, #32 64: \type d18, d19, q0, q1, q2, q3 \type d20, d21, q0, q1, q2, q3 \type d22, d23, q0, q1, q2, q3 vst1.8 {q8, q9}, [r0, :128], r1 \type d16, d17, q0, q1, q2, q3 vst1.8 {q10, q11}, [r6, :128], r1 \type d18, d19, q0, q1, q2, q3 \type d20, d21, q0, q1, q2, q3 vst1.8 {q8, q9}, [r0, :128], r1 \type d22, d23, q0, q1, q2, q3 subs r5, r5, #2 vst1.8 {q10, q11}, [r6, :128], r1 ble 0f \type d16, d17, q0, q1, q2, q3 b 64b 1280: sub r1, r1, #32 add r6, r0, #64 128: \type d18, d19, q0, q1, q2, q3 \type d20, d21, q0, q1, q2, q3 \type d22, d23, q0, q1, q2, q3 vst1.8 {q8, q9}, [r0, :128]! \type d16, d17, q0, q1, q2, q3 vst1.8 {q10, q11}, [r0, :128], r1 \type d18, d19, q0, q1, q2, q3 \type d20, d21, q0, q1, q2, q3 vst1.8 {q8, q9}, [r6, :128]! \type d22, d23, q0, q1, q2, q3 subs r5, r5, #1 vst1.8 {q10, q11}, [r6, :128], r1 ble 0f \type d16, d17, q0, q1, q2, q3 b 128b 0: pop {r4-r6,pc} endfunc .endm bidir_fn avg bidir_fn w_avg bidir_fn mask .macro w_mask_fn type function w_mask_\type\()_8bpc_neon, export=1 push {r4-r9,lr} ldrd r4, r5, [sp, #28] ldrd r6, r7, [sp, #36] clz r8, r4 adr r9, L(w_mask_\type\()_tbl) sub r8, r8, #24 ldr r8, [r9, r8, lsl #2] add r9, r9, r8 movw r12, #6903 vdup.16 q14, r12 .if \type == 444 vmov.i8 q15, #64 .elseif \type == 422 vdup.8 d0, r7 // d0[] <- sign vmov.i8 d30, #129 vsub.i8 d30, d30, d0 // 129 - sign .elseif \type == 420 vdup.16 q0, r7 // d0[] <- sign vmov.i16 q15, #256 vsub.i16 q15, q15, q0 // 256 - sign .endif add r12, r0, r1 lsl r1, r1, #1 bx r9 .align 2 L(w_mask_\type\()_tbl): .word 1280f - L(w_mask_\type\()_tbl) + CONFIG_THUMB .word 640f - L(w_mask_\type\()_tbl) + CONFIG_THUMB .word 320f - L(w_mask_\type\()_tbl) + CONFIG_THUMB .word 160f - L(w_mask_\type\()_tbl) + CONFIG_THUMB .word 8f - L(w_mask_\type\()_tbl) + CONFIG_THUMB .word 4f - L(w_mask_\type\()_tbl) + CONFIG_THUMB 4: vld1.16 {d0, d1, d2, d3}, [r2, :128]! // tmp1 (four rows at once) vld1.16 {d4, d5, d6, d7}, [r3, :128]! // tmp2 (four rows at once) subs r5, r5, #4 vsub.i16 q8, q2, q0 // tmp2-tmp1 vsub.i16 q9, q3, q1 vabd.s16 q10, q0, q2 // (abs(tmp1[x] - tmp2[x])) vabd.s16 q11, q1, q3 vqsub.u16 q10, q14, q10 // 6903 - abs () vqsub.u16 q11, q14, q11 vshr.s16 q10, q10, #8 // 64-m = (6903 - abs()) >> 8 vshr.s16 q11, q11, #8 vshl.s16 q12, q10, #9 // (64-m)<<9 vshl.s16 q13, q11, #9 vqdmulh.s16 q12, q12, q8 // ((tmp2-tmp1)*(64-m)<<9)>>15 vqdmulh.s16 q13, q13, q9 vadd.i16 q12, q12, q0 // (((tmp2-tmp1)*(64-m)<<9)>>15) + tmp1 vadd.i16 q13, q13, q1 vqrshrun.s16 d24, q12, #4 // (((((tmp2-tmp1)*(64-m)<<9)>>15) + tmp1) + 8) >> 4 vqrshrun.s16 d25, q13, #4 .if \type == 444 vmovn.u16 d20, q10 // 64 - m vmovn.u16 d21, q11 vsub.i8 q10, q15, q10 // m vst1.8 {d20, d21}, [r6, :128]! .elseif \type == 422 vpadd.s16 d20, d20, d21 // (64 - m) + (64 - n) (column wise addition) vpadd.s16 d21, d22, d23 vmovn.s16 d6, q10 vhsub.u8 d6, d30, d6 // ((129 - sign) - ((64 - m) + (64 - n))) >> 1 vst1.8 {d6}, [r6, :64]! .elseif \type == 420 vadd.s16 d20, d20, d21 // (64 - my1) + (64 - my2) (row wise addition) vadd.s16 d21, d22, d23 vpadd.s16 d20, d20, d21 // (128 - m) + (128 - n) (column wise addition) vsub.s16 d20, d30, d20 // (256 - sign) - ((128 - m) + (128 - n)) vrshrn.u16 d20, q10, #2 // ((256 - sign) - ((128 - m) + (128 - n)) + 2) >> 2 vst1.32 {d20[0]}, [r6, :32]! .endif vst1.32 {d24[0]}, [r0, :32], r1 vst1.32 {d24[1]}, [r12, :32], r1 vst1.32 {d25[0]}, [r0, :32], r1 vst1.32 {d25[1]}, [r12, :32], r1 bgt 4b pop {r4-r9,pc} 8: vld1.16 {d0, d1, d2, d3}, [r2, :128]! // tmp1y1, tmp1y2 vld1.16 {d4, d5, d6, d7}, [r3, :128]! // tmp2y1, tmp2y2 subs r5, r5, #2 vsub.i16 q8, q2, q0 // tmp2y1 - tmp1y1 vsub.i16 q9, q3, q1 // tmp2y2 - tmp1y2 vabd.s16 q10, q0, q2 // abs(tmp1y1 - tmp2y1) vabd.s16 q11, q1, q3 // abs(tmp1y2 - tmp2y2) vqsub.u16 q10, q14, q10 // 6903 - abs(tmp1y1 - tmp2y1) vqsub.u16 q11, q14, q11 // 6903 - abs(tmp1y2 - tmp2y2) vshr.s16 q10, q10, #8 // 64 - my1 = 6903 - abs(tmp1y1 - tmp2y1) >> 8 vshr.s16 q11, q11, #8 // 64 - my2 = 6903 - abs(tmp1y2 - tmp2y2) >> 8 vshl.s16 q12, q10, #9 // (64 - my1) << 9 vshl.s16 q13, q11, #9 // (64 - my2) << 9 vqdmulh.s16 q12, q12, q8 // ((tmp2y1 - tmp1y1) * (64 - my1) << 9) >> 15 vqdmulh.s16 q13, q13, q9 // ((tmp2y2 - tmp1y2) * (64 - my2) << 9) >> 15 vadd.s16 q12, q12, q0 // (((tmp2y1 - tmp1y1) * (64 - my1) << 9) >> 15) + tmp1y1 vadd.s16 q13, q13, q1 // (((tmp2y2 - tmp1y2) * (64 - my2) << 9) >> 15) + tmp1y2 vqrshrun.s16 d24, q12, #4 // (((((tmp2y1 - tmp1y1) * (64 - my1) << 9) >> 15) + tmp1y1) + 8) >> 4 vqrshrun.s16 d25, q13, #4 // (((((tmp2y2 - tmp1y2) * (64 - my2) << 9) >> 15) + tmp1y2) + 8) >> 4 .if \type == 444 vmovn.u16 d20, q10 // 64 - m vmovn.u16 d21, q11 vsub.i8 q10, q15, q10 // m vst1.8 {d20, d21}, [r6, :128]! .elseif \type == 422 vpadd.s16 d20, d20, d21 // (64 - my1) + (64 - ny1) (column wise addition) vpadd.s16 d21, d22, d23 // (64 - my2) + (64 - ny2) vmovn.s16 d20, q10 vhsub.u8 d20, d30, d20 // ((129 - sign) - ((64 - my1/y2) + (64 - ny1/y2))) >> 1 vst1.8 {d20}, [r6, :64]! .elseif \type == 420 vadd.s16 q10, q10, q11 // (64 - my1) + (64 - my2) (row wise addition) vpadd.s16 d20, d20, d21 // (128 - m) + (128 - n) (column wise addition) vsub.s16 d20, d30, d20 // (256 - sign) - ((128 - m) + (128 - n)) vrshrn.u16 d20, q10, #2 // ((256 - sign) - ((128 - m) + (128 - n)) + 2) >> 2 vst1.32 {d20[0]}, [r6, :32]! .endif vst1.16 {d24}, [r0, :64], r1 vst1.16 {d25}, [r12, :64], r1 bgt 8b pop {r4-r9,pc} 1280: 640: 320: 160: sub r1, r1, r4 .if \type == 444 add lr, r6, r4 .elseif \type == 422 add lr, r6, r4, lsr #1 .endif add r9, r3, r4, lsl #1 add r7, r2, r4, lsl #1 161: mov r8, r4 16: vld1.16 {d0, d1, d2, d3}, [r2, :128]! // tmp1y1 vld1.16 {d4, d5, d6, d7}, [r3, :128]! // tmp2y1 vld1.16 {d16, d17, d18, d19}, [r7, :128]! // tmp1y2 subs r8, r8, #16 vsub.i16 q2, q2, q0 // tmp2y1 - tmp1y1 vsub.i16 q3, q3, q1 vabs.s16 q10, q2 // abs(tm2y1 - tmp1y1) vabs.s16 q11, q3 vqsub.u16 q10, q14, q10 // 6903 - abs(tmp1y1 - tmp2y1) vqsub.u16 q11, q14, q11 vshr.s16 q10, q10, #8 // 64 - my1 = 6903 - abs(tmp1y1 - tmp2y1) >> 8 vshr.s16 q11, q11, #8 vshl.s16 q12, q10, #9 // (64 - my1) << 9 vshl.s16 q13, q11, #9 vqdmulh.s16 q12, q12, q2 // ((tmp2y1 - tmp1y1) * (64 - my1) << 9) >> 15 vqdmulh.s16 q13, q13, q3 vadd.i16 q12, q12, q0 // (((tmp2y1 - tmp1y1) * (64 - my1) << 9) >> 15) + tmp1y1 vadd.i16 q13, q13, q1 vld1.16 {d0, d1, d2, d3}, [r9, :128]! // tmp2h2 .if \type == 444 vmovn.u16 d20, q10 // 64 - my1 vmovn.u16 d21, q11 vsub.i8 q10, q15, q10 // my1 vst1.8 {d20, d21}, [r6, :128]! .elseif \type == 422 vpadd.s16 d20, d20, d21 // (64 - my1) + (64 - ny1) (column wise addition) vpadd.s16 d21, d22, d23 vmovn.s16 d20, q10 vhsub.u8 d20, d30, d20 // ((129 - sign) - ((64 - my1) + (64 - ny1))) >> 1 vst1.8 {d20}, [r6, :64]! .endif vqrshrun.s16 d24, q12, #4 // (((((tmp2y1 - tmp1y1)*(64 - my1) << 9) >> 15) + tmp1y1) + 8) >> 4 vqrshrun.s16 d25, q13, #4 vsub.i16 q0, q0, q8 // tmp2y2 - tmp1y2 vsub.i16 q1, q1, q9 vst1.16 {d24, d25}, [r0, :128]! // store dsty1 vabs.s16 q2, q0 // abs(tmp2y2 - tmp1y2) vabs.s16 q3, q1 vqsub.u16 q2, q14, q2 // 6903 - abs(tmp2y2 - tmp1y2) vqsub.u16 q3, q14, q3 vshr.s16 q2, q2, #8 // (6903 - abs(tmp2y2 - tmp1y2)) >> 8 vshr.s16 q3, q3, #8 vshl.s16 q12, q2, #9 // (64 - my2) << 9 vshl.s16 q13, q3, #9 .if \type == 444 vmovn.u16 d4, q2 // 64 - my2 vmovn.u16 d5, q3 vsub.i8 q2, q15, q2 // my2 vst1.8 {d4, d5}, [lr, :128]! .elseif \type == 422 vpadd.s16 d4, d4, d5 // (64 - my2) + (64 - ny2) (column wise addition) vpadd.s16 d5, d6, d7 vmovn.s16 d4, q2 vhsub.u8 d4, d30, d4 // ((129 - sign) - ((64 - my2) + (64 - ny2))) >> 1 vst1.8 {d4}, [lr, :64]! .elseif \type == 420 vadd.s16 q10, q10, q2 // (64 - my1) + (64 - my2) (row wise addition) vadd.s16 q11, q11, q3 vpadd.s16 d20, d20, d21 // (128 - m) + (128 - n) (column wise addition) vpadd.s16 d21, d22, d23 vsub.s16 q10, q15, q10 // (256 - sign) - ((128 - m) + (128 - n)) vrshrn.u16 d20, q10, #2 // ((256 - sign) - ((128 - m) + (128 - n)) + 2) >> 2 vst1.8 {d20}, [r6, :64]! .endif vqdmulh.s16 q12, q12, q0 // ((tmp2y2 - tmp1y2) * (64 - my2) << 9) >> 15 vqdmulh.s16 q13, q13, q1 vadd.i16 q12, q12, q8 // (((tmp2y2 - tmp1y2) * (64 - my2) << 9) >> 15) + tmp1y2 vadd.i16 q13, q13, q9 vqrshrun.s16 d24, q12, #4 // (((((tmp2y2 - tmp1y2)*(64 - my2) << 9) >> 15) + tmp1y2) + 8) >> 4 vqrshrun.s16 d25, q13, #4 vst1.16 {d24, d25}, [r12, :128]! // store dsty2 bgt 16b subs r5, r5, #2 add r2, r2, r4, lsl #1 add r3, r3, r4, lsl #1 add r7, r7, r4, lsl #1 add r9, r9, r4, lsl #1 .if \type == 444 add r6, r6, r4 add lr, lr, r4 .elseif \type == 422 add r6, r6, r4, lsr #1 add lr, lr, r4, lsr #1 .endif add r0, r0, r1 add r12, r12, r1 bgt 161b pop {r4-r9,pc} endfunc .endm w_mask_fn 444 w_mask_fn 422 w_mask_fn 420 function blend_8bpc_neon, export=1 push {r4-r5,lr} ldrd r4, r5, [sp, #12] clz lr, r3 adr r3, L(blend_tbl) sub lr, lr, #26 ldr lr, [r3, lr, lsl #2] add r3, r3, lr bx r3 .align 2 L(blend_tbl): .word 320f - L(blend_tbl) + CONFIG_THUMB .word 160f - L(blend_tbl) + CONFIG_THUMB .word 80f - L(blend_tbl) + CONFIG_THUMB .word 40f - L(blend_tbl) + CONFIG_THUMB 40: vmov.i8 d22, #64 add r12, r0, r1 lsl r1, r1, #1 4: vld1.u8 {d2}, [r5, :64]! vld1.u8 {d1}, [r2, :64]! vld1.32 {d0[]}, [r0, :32] subs r4, r4, #2 vld1.32 {d0[1]}, [r12, :32] vsub.i8 d3, d22, d2 vmull.u8 q8, d1, d2 vmlal.u8 q8, d0, d3 vrshrn.i16 d20, q8, #6 vst1.32 {d20[0]}, [r0, :32], r1 vst1.32 {d20[1]}, [r12, :32], r1 bgt 4b pop {r4-r5,pc} 80: vmov.i8 d16, #64 add r12, r0, r1 lsl r1, r1, #1 8: vld1.u8 {q1}, [r5, :128]! vld1.u8 {q2}, [r2, :128]! vld1.u8 {d0}, [r0, :64] vsub.i8 d17, d16, d2 vld1.u8 {d1}, [r12, :64] subs r4, r4, #2 vsub.i8 d18, d16, d3 vmull.u8 q3, d2, d4 vmlal.u8 q3, d0, d17 vmull.u8 q10, d3, d5 vmlal.u8 q10, d1, d18 vrshrn.i16 d22, q3, #6 vrshrn.i16 d23, q10, #6 vst1.u8 {d22}, [r0, :64], r1 vst1.u8 {d23}, [r12, :64], r1 bgt 8b pop {r4-r5,pc} 160: vmov.i8 q12, #64 add r12, r0, r1 lsl r1, r1, #1 16: vld1.u8 {q1, q2}, [r5, :128]! vld1.u8 {q8, q9}, [r2, :128]! vld1.u8 {q0}, [r0, :128] subs r4, r4, #2 vsub.i8 q15, q12, q1 vld1.u8 {q13}, [r12, :128] vmull.u8 q3, d16, d2 vmlal.u8 q3, d0, d30 vmull.u8 q14, d17, d3 vmlal.u8 q14, d1, d31 vsub.i8 q15, q12, q2 vrshrn.i16 d20, q3, #6 vrshrn.i16 d21, q14, #6 vmull.u8 q3, d18, d4 vmlal.u8 q3, d26, d30 vmull.u8 q14, d19, d5 vmlal.u8 q14, d27, d31 vrshrn.i16 d22, q3, #6 vrshrn.i16 d23, q14, #6 vst1.u8 {q10}, [r0, :128], r1 vst1.u8 {q11}, [r12, :128], r1 bgt 16b pop {r4-r5,pc} 320: vmov.i8 q10, #64 32: vld1.u8 {q2, q3}, [r5, :128]! vld1.u8 {q8, q9}, [r2, :128]! vld1.u8 {q0, q1}, [r0, :128] subs r4, r4, #1 vsub.i8 q11, q10, q2 vmull.u8 q15, d16, d4 vmlal.u8 q15, d0, d22 vmull.u8 q14, d17, d5 vmlal.u8 q14, d1, d23 vsub.i8 q11, q10, q3 vrshrn.i16 d24, q15, #6 vrshrn.i16 d25, q14, #6 vmull.u8 q15, d18, d6 vmlal.u8 q15, d2, d22 vmull.u8 q14, d19, d7 vmlal.u8 q14, d3, d23 vrshrn.i16 d26, q15, #6 vrshrn.i16 d27, q14, #6 vst1.u8 {q12, q13}, [r0, :128], r1 bgt 32b pop {r4-r5,pc} endfunc function blend_h_8bpc_neon, export=1 push {r4-r5,lr} ldr r4, [sp, #12] movrel r5, X(obmc_masks) add r5, r5, r4 sub r4, r4, r4, lsr #2 clz lr, r3 adr r12, L(blend_h_tbl) sub lr, lr, #24 ldr lr, [r12, lr, lsl #2] add r12, r12, lr bx r12 .align 2 L(blend_h_tbl): .word 1280f - L(blend_h_tbl) + CONFIG_THUMB .word 640f - L(blend_h_tbl) + CONFIG_THUMB .word 320f - L(blend_h_tbl) + CONFIG_THUMB .word 160f - L(blend_h_tbl) + CONFIG_THUMB .word 80f - L(blend_h_tbl) + CONFIG_THUMB .word 40f - L(blend_h_tbl) + CONFIG_THUMB .word 20f - L(blend_h_tbl) + CONFIG_THUMB 20: vmov.i8 d22, #64 add r12, r0, r1 lsl r1, r1, #1 2: vld1.16 {d2[], d3[]}, [r5, :16]! vld1.32 {d1[]}, [r2, :32]! subs r4, r4, #2 vld1.16 {d0[]}, [r0, :16] vzip.8 d2, d3 vsub.i8 d4, d22, d2 vld1.16 {d0[1]}, [r12, :16] vmull.u8 q8, d1, d2 vmlal.u8 q8, d0, d4 vrshrn.i16 d20, q8, #6 vst1.16 {d20[0]}, [r0, :16], r1 vst1.16 {d20[1]}, [r12, :16], r1 bgt 2b pop {r4-r5,pc} 40: vmov.i8 d22, #64 add r12, r0, r1 lsl r1, r1, #1 4: vld2.u8 {d2[], d3[]}, [r5, :16]! vld1.u8 {d1}, [r2, :64]! subs r4, r4, #2 vext.u8 d2, d2, d3, #4 vld1.32 {d0[]}, [r0, :32] vsub.i8 d6, d22, d2 vld1.32 {d0[1]}, [r12, :32] vmull.u8 q8, d1, d2 vmlal.u8 q8, d0, d6 vrshrn.i16 d20, q8, #6 vst1.32 {d20[0]}, [r0, :32], r1 vst1.32 {d20[1]}, [r12, :32], r1 bgt 4b pop {r4-r5,pc} 80: vmov.i8 q8, #64 add r12, r0, r1 lsl r1, r1, #1 8: vld2.u8 {d2[], d3[]}, [r5, :16]! vld1.u8 {d4, d5}, [r2, :128]! vld1.u8 {d0}, [r0, :64] vsub.i8 q9, q8, q1 vld1.u8 {d1}, [r12, :64] subs r4, r4, #2 vmull.u8 q3, d2, d4 vmlal.u8 q3, d0, d18 vmull.u8 q10, d3, d5 vmlal.u8 q10, d1, d19 vrshrn.i16 d22, q3, #6 vrshrn.i16 d23, q10, #6 vst1.u8 {d22}, [r0, :64], r1 vst1.u8 {d23}, [r12, :64], r1 bgt 8b pop {r4-r5,pc} 160: vmov.i8 q12, #64 add r12, r0, r1 lsl r1, r1, #1 16: vld2.u8 {d28[], d29[]}, [r5, :16]! vld1.u8 {d2, d3, d4, d5}, [r2, :128]! vsub.i8 q15, q12, q14 vld1.u8 {q0}, [r0, :128] subs r4, r4, #2 vld1.u8 {q13}, [r12, :128] vmull.u8 q3, d2, d28 vmlal.u8 q3, d0, d30 vmull.u8 q8, d3, d28 vmlal.u8 q8, d1, d30 vrshrn.i16 d18, q3, #6 vrshrn.i16 d19, q8, #6 vmull.u8 q3, d4, d29 vmlal.u8 q3, d26, d31 vmull.u8 q8, d5, d29 vmlal.u8 q8, d27, d31 vrshrn.i16 d20, q3, #6 vrshrn.i16 d21, q8, #6 vst1.u8 {q9}, [r0, :128], r1 vst1.u8 {q10}, [r12, :128], r1 bgt 16b pop {r4-r5,pc} 320: 640: 1280: vmov.i8 d20, #64 sub r1, r1, r3 321: vld1.u8 {d6[]}, [r5]! vsub.i8 d7, d20, d6 mov r12, r3 32: vld1.u8 {q8, q9}, [r2, :128]! vld1.u8 {q0, q1}, [r0, :128] vmull.u8 q15, d16, d6 vmlal.u8 q15, d0, d7 vmull.u8 q14, d17, d6 vmlal.u8 q14, d1, d7 vrshrn.i16 d0, q15, #6 vrshrn.i16 d1, q14, #6 vmull.u8 q15, d18, d6 vmlal.u8 q15, d2, d7 vmull.u8 q14, d19, d6 vmlal.u8 q14, d3, d7 vrshrn.i16 d2, q15, #6 vrshrn.i16 d3, q14, #6 subs r12, r12, #32 vst1.u8 {q0, q1}, [r0, :128]! bgt 32b add r0, r0, r1 subs r4, r4, #1 bgt 321b pop {r4-r5,pc} endfunc function blend_v_8bpc_neon, export=1 push {r4,lr} ldr r4, [sp, #8] movrel lr, X(obmc_masks) add lr, lr, r3 clz r12, r3 adr r3, L(blend_v_tbl) sub r12, r12, #26 ldr r12, [r3, r12, lsl #2] add r3, r3, r12 bx r3 .align 2 L(blend_v_tbl): .word 320f - L(blend_v_tbl) + CONFIG_THUMB .word 160f - L(blend_v_tbl) + CONFIG_THUMB .word 80f - L(blend_v_tbl) + CONFIG_THUMB .word 40f - L(blend_v_tbl) + CONFIG_THUMB .word 20f - L(blend_v_tbl) + CONFIG_THUMB 20: vmov.i8 d22, #64 vld1.8 {d2[]}, [lr] add r12, r0, r1 lsl r1, r1, #1 vsub.i8 d3, d22, d2 2: vld1.16 {d1[0]}, [r2, :16]! vld1.8 {d0[]}, [r0] subs r4, r4, #2 vld1.8 {d1[1]}, [r2] vld1.8 {d0[1]}, [r12] vmull.u8 q2, d1, d2 vmlal.u8 q2, d0, d3 vrshrn.i16 d6, q2, #6 add r2, r2, #2 vst1.8 {d6[0]}, [r0], r1 vst1.8 {d6[1]}, [r12], r1 bgt 2b pop {r4,pc} 40: vmov.i8 d22, #64 vld1.32 {d4[]}, [lr, :32] add r12, r0, r1 lsl r1, r1, #1 vsub.i8 d5, d22, d4 sub r1, r1, #2 4: vld1.u8 {d2}, [r2, :64]! vld1.32 {d0[]}, [r0, :32] vld1.32 {d0[1]}, [r12, :32] subs r4, r4, #2 vmull.u8 q3, d2, d4 vmlal.u8 q3, d0, d5 vrshrn.i16 d20, q3, #6 vst1.16 {d20[0]}, [r0, :16]! vst1.16 {d20[2]}, [r12, :16]! vst1.8 {d20[2]}, [r0], r1 vst1.8 {d20[6]}, [r12], r1 bgt 4b pop {r4,pc} 80: vmov.i8 d16, #64 vld1.u8 {d2}, [lr, :64] add r12, r0, r1 lsl r1, r1, #1 vsub.i8 d17, d16, d2 sub r1, r1, #4 8: vld1.u8 {d4, d5}, [r2, :128]! vld1.u8 {d0}, [r0, :64] vld1.u8 {d1}, [r12, :64] subs r4, r4, #2 vmull.u8 q3, d2, d4 vmlal.u8 q3, d0, d17 vmull.u8 q10, d2, d5 vmlal.u8 q10, d1, d17 vrshrn.i16 d22, q3, #6 vrshrn.i16 d23, q10, #6 vst1.32 {d22[0]}, [r0, :32]! vst1.32 {d23[0]}, [r12, :32]! vst1.16 {d22[2]}, [r0, :16], r1 vst1.16 {d23[2]}, [r12, :16], r1 bgt 8b pop {r4,pc} 160: vmov.i8 q12, #64 vld1.u8 {q14}, [lr, :128] add r12, r0, r1 lsl r1, r1, #1 vsub.i8 q11, q12, q14 sub r1, r1, #8 16: vld1.u8 {q1, q2}, [r2, :128]! vld1.u8 {q0}, [r0, :128] subs r4, r4, #2 vld1.u8 {q13}, [r12, :128] vmull.u8 q3, d2, d28 vmlal.u8 q3, d0, d22 vmull.u8 q8, d3, d29 vmlal.u8 q8, d1, d23 vrshrn.i16 d18, q3, #6 vrshrn.i16 d19, q8, #6 vmull.u8 q3, d4, d28 vmlal.u8 q3, d26, d22 vmull.u8 q8, d5, d29 vmlal.u8 q8, d27, d23 vrshrn.i16 d20, q3, #6 vrshrn.i16 d21, q8, #6 vst1.u8 {d18}, [r0, :64]! vst1.u8 {d20}, [r12, :64]! vst1.32 {d19[0]}, [r0, :32], r1 vst1.32 {d21[0]}, [r12, :32], r1 bgt 16b pop {r4,pc} 320: vmov.i8 q10, #64 vld1.u8 {q2, q3}, [lr, :128] vsub.i8 q11, q10, q2 vsub.i8 d24, d20, d6 32: vld1.u8 {q8, q9}, [r2, :128]! vld1.u8 {d0, d1, d2}, [r0, :64] subs r4, r4, #1 vmull.u8 q15, d16, d4 vmlal.u8 q15, d0, d22 vmull.u8 q14, d17, d5 vmlal.u8 q14, d1, d23 vrshrn.i16 d0, q15, #6 vrshrn.i16 d1, q14, #6 vmull.u8 q15, d18, d6 vmlal.u8 q15, d2, d24 vrshrn.i16 d2, q15, #6 vst1.u8 {d0, d1, d2}, [r0, :64], r1 bgt 32b pop {r4,pc} endfunc // This has got the same signature as the put_8tap functions, // assumes that the caller has loaded the h argument into r5, // and assumes that r8 is set to (clz(w)-24). function put_neon adr r9, L(put_tbl) ldr r8, [r9, r8, lsl #2] add r9, r9, r8 bx r9 .align 2 L(put_tbl): .word 1280f - L(put_tbl) + CONFIG_THUMB .word 640f - L(put_tbl) + CONFIG_THUMB .word 32f - L(put_tbl) + CONFIG_THUMB .word 160f - L(put_tbl) + CONFIG_THUMB .word 8f - L(put_tbl) + CONFIG_THUMB .word 4f - L(put_tbl) + CONFIG_THUMB .word 2f - L(put_tbl) + CONFIG_THUMB 2: vld1.16 {d0[]}, [r2], r3 vld1.16 {d1[]}, [r2], r3 subs r5, r5, #2 vst1.16 {d0[0]}, [r0, :16], r1 vst1.16 {d1[0]}, [r0, :16], r1 bgt 2b pop {r4-r11,pc} 4: vld1.32 {d0[]}, [r2], r3 vld1.32 {d1[]}, [r2], r3 subs r5, r5, #2 vst1.32 {d0[0]}, [r0, :32], r1 vst1.32 {d1[0]}, [r0, :32], r1 bgt 4b pop {r4-r11,pc} 8: vld1.8 {d0}, [r2], r3 vld1.8 {d1}, [r2], r3 subs r5, r5, #2 vst1.8 {d0}, [r0, :64], r1 vst1.8 {d1}, [r0, :64], r1 bgt 8b pop {r4-r11,pc} 160: add r8, r0, r1 lsl r1, r1, #1 add r9, r2, r3 lsl r3, r3, #1 16: vld1.8 {q0}, [r2], r3 vld1.8 {q1}, [r9], r3 subs r5, r5, #2 vst1.8 {q0}, [r0, :128], r1 vst1.8 {q1}, [r8, :128], r1 bgt 16b pop {r4-r11,pc} 32: vld1.8 {q0, q1}, [r2], r3 subs r5, r5, #1 vst1.8 {q0, q1}, [r0, :128], r1 bgt 32b pop {r4-r11,pc} 640: sub r1, r1, #32 sub r3, r3, #32 64: vld1.8 {q0, q1}, [r2]! vst1.8 {q0, q1}, [r0, :128]! vld1.8 {q2, q3}, [r2], r3 subs r5, r5, #1 vst1.8 {q2, q3}, [r0, :128], r1 bgt 64b pop {r4-r11,pc} 1280: sub r1, r1, #96 sub r3, r3, #96 128: vld1.8 {q8, q9}, [r2]! vst1.8 {q8, q9}, [r0, :128]! vld1.8 {q10, q11}, [r2]! vst1.8 {q10, q11}, [r0, :128]! vld1.8 {q12, q13}, [r2]! vst1.8 {q12, q13}, [r0, :128]! vld1.8 {q14, q15}, [r2], r3 subs r5, r5, #1 vst1.8 {q14, q15}, [r0, :128], r1 bgt 128b pop {r4-r11,pc} endfunc // This has got the same signature as the put_8tap functions, // assumes that the caller has loaded the h argument into r4, // and assumes that r8 is set to (clz(w)-24), and r7 to w*2. function prep_neon adr r9, L(prep_tbl) ldr r8, [r9, r8, lsl #2] add r9, r9, r8 bx r9 .align 2 L(prep_tbl): .word 1280f - L(prep_tbl) + CONFIG_THUMB .word 640f - L(prep_tbl) + CONFIG_THUMB .word 320f - L(prep_tbl) + CONFIG_THUMB .word 160f - L(prep_tbl) + CONFIG_THUMB .word 8f - L(prep_tbl) + CONFIG_THUMB .word 4f - L(prep_tbl) + CONFIG_THUMB 4: vld1.32 {d0[]}, [r1], r2 vld1.32 {d2[]}, [r1], r2 subs r4, r4, #2 vshll.u8 q0, d0, #4 vshll.u8 q1, d2, #4 vst1.16 {d1, d2}, [r0, :64]! bgt 4b pop {r4-r11,pc} 8: vld1.8 {d0}, [r1], r2 vld1.8 {d2}, [r1], r2 subs r4, r4, #2 vshll.u8 q0, d0, #4 vshll.u8 q1, d2, #4 vst1.16 {q0, q1}, [r0, :128]! bgt 8b pop {r4-r11,pc} 160: add r9, r1, r2 lsl r2, r2, #1 add r8, r0, r7 lsl r7, r7, #1 16: vld1.8 {q2}, [r1], r2 vld1.8 {q3}, [r9], r2 subs r4, r4, #2 vshll.u8 q0, d4, #4 vshll.u8 q1, d5, #4 vshll.u8 q2, d6, #4 vshll.u8 q3, d7, #4 vst1.16 {q0, q1}, [r0, :128], r7 vst1.16 {q2, q3}, [r8, :128], r7 bgt 16b pop {r4-r11,pc} 320: add r8, r0, r3 32: vld1.8 {q0, q1}, [r1], r2 subs r4, r4, #2 vshll.u8 q8, d0, #4 vshll.u8 q9, d1, #4 vld1.8 {q2, q3}, [r1], r2 vshll.u8 q10, d2, #4 vshll.u8 q11, d3, #4 vshll.u8 q12, d4, #4 vst1.16 {q8, q9}, [r0, :128], r7 vshll.u8 q13, d5, #4 vst1.16 {q10, q11}, [r8, :128], r7 vshll.u8 q14, d6, #4 vst1.16 {q12, q13}, [r0, :128], r7 vshll.u8 q15, d7, #4 vst1.16 {q14, q15}, [r8, :128], r7 bgt 32b pop {r4-r11,pc} 640: sub r2, r2, #32 add r8, r0, #32 mov r6, #64 64: vld1.8 {q0, q1}, [r1]! subs r4, r4, #1 vshll.u8 q8, d0, #4 vshll.u8 q9, d1, #4 vld1.8 {q2, q3}, [r1], r2 vshll.u8 q10, d2, #4 vshll.u8 q11, d3, #4 vshll.u8 q12, d4, #4 vst1.16 {q8, q9}, [r0, :128], r6 vshll.u8 q13, d5, #4 vshll.u8 q14, d6, #4 vst1.16 {q10, q11}, [r8, :128], r6 vshll.u8 q15, d7, #4 vst1.16 {q12, q13}, [r0, :128], r6 vst1.16 {q14, q15}, [r8, :128], r6 bgt 64b pop {r4-r11,pc} 1280: sub r2, r2, #96 add r8, r0, #32 mov r6, #64 128: vld1.8 {q0, q1}, [r1]! vld1.8 {q2, q3}, [r1]! vshll.u8 q10, d0, #4 vshll.u8 q11, d1, #4 vshll.u8 q12, d2, #4 vshll.u8 q13, d3, #4 vshll.u8 q14, d4, #4 vshll.u8 q15, d5, #4 vld1.8 {q8, q9}, [r1]! vst1.16 {q10, q11}, [r0, :128], r6 vst1.16 {q12, q13}, [r8, :128], r6 vshll.u8 q0, d6, #4 vshll.u8 q1, d7, #4 vshll.u8 q2, d16, #4 vshll.u8 q3, d17, #4 vshll.u8 q8, d18, #4 vshll.u8 q9, d19, #4 vld1.8 {q10, q11}, [r1], r2 vst1.16 {q14, q15}, [r0, :128], r6 vst1.16 {q0, q1}, [r8, :128], r6 vshll.u8 q12, d20, #4 vshll.u8 q13, d21, #4 vshll.u8 q14, d22, #4 vshll.u8 q15, d23, #4 subs r4, r4, #1 vst1.16 {q2, q3}, [r0, :128], r6 vst1.16 {q8, q9}, [r8, :128], r6 vst1.16 {q12, q13}, [r0, :128], r6 vst1.16 {q14, q15}, [r8, :128], r6 bgt 128b pop {r4-r11,pc} endfunc .macro load_slice s0, s1, strd, wd, d0, d1, d2, d3, d4, d5, d6 vld1.\wd {\d0[]}, [\s0], \strd vld1.\wd {\d1[]}, [\s1], \strd .ifnb \d2 vld1.\wd {\d2[]}, [\s0], \strd vld1.\wd {\d3[]}, [\s1], \strd .endif .ifnb \d4 vld1.\wd {\d4[]}, [\s0], \strd .endif .ifnb \d5 vld1.\wd {\d5[]}, [\s1], \strd .endif .ifnb \d6 vld1.\wd {\d6[]}, [\s0], \strd .endif .endm .macro load_reg s0, s1, strd, d0, d1, d2, d3, d4, d5, d6 vld1.8 {\d0}, [\s0], \strd vld1.8 {\d1}, [\s1], \strd .ifnb \d2 vld1.8 {\d2}, [\s0], \strd vld1.8 {\d3}, [\s1], \strd .endif .ifnb \d4 vld1.8 {\d4}, [\s0], \strd .endif .ifnb \d5 vld1.8 {\d5}, [\s1], \strd .endif .ifnb \d6 vld1.8 {\d6}, [\s0], \strd .endif .endm .macro load_16 s0, s1, strd, d0, d1, d2, d3, d4, d5, d6 load_slice \s0, \s1, \strd, 16, \d0, \d1, \d2, \d3, \d4, \d5, \d6 .endm .macro load_32 s0, s1, strd, d0, d1, d2, d3, d4, d5, d6 load_slice \s0, \s1, \strd, 32, \d0, \d1, \d2, \d3, \d4, \d5, \d6 .endm .macro interleave_1_16 r0, r1, r2, r3, r4 vext.8 \r0, \r0, \r1, #6 vext.8 \r1, \r1, \r2, #6 .ifnb \r3 vext.8 \r2, \r2, \r3, #6 vext.8 \r3, \r3, \r4, #6 .endif .endm .macro interleave_1_32 r0, r1, r2, r3, r4 vext.8 \r0, \r0, \r1, #4 vext.8 \r1, \r1, \r2, #4 .ifnb \r3 vext.8 \r2, \r2, \r3, #4 vext.8 \r3, \r3, \r4, #4 .endif .endm .macro vmovl_u8 q0, d0, q1, d1, q2, d2, q3, d3, q4, d4, q5, d5, q6, d6 vmovl.u8 \q0, \d0 vmovl.u8 \q1, \d1 .ifnb \q2 vmovl.u8 \q2, \d2 vmovl.u8 \q3, \d3 .endif .ifnb \q4 vmovl.u8 \q4, \d4 .endif .ifnb \q5 vmovl.u8 \q5, \d5 .endif .ifnb \q6 vmovl.u8 \q6, \d6 .endif .endm .macro mul_mla_4 d, s0, s1, s2, s3 vmul.s16 \d, \s0, d0[0] vmla.s16 \d, \s1, d0[1] vmla.s16 \d, \s2, d0[2] vmla.s16 \d, \s3, d0[3] .endm .macro mul_mla_8_0 d0, s0, s1, s2, s3, s4, s5, s6, s7 vmul.s16 \d0, \s0, d0[0] vmla.s16 \d0, \s1, d0[1] vmla.s16 \d0, \s2, d0[2] vmla.s16 \d0, \s3, d0[3] vmla.s16 \d0, \s4, d1[0] vmla.s16 \d0, \s5, d1[1] vmla.s16 \d0, \s6, d1[2] vmla.s16 \d0, \s7, d1[3] .endm .macro mul_mla_8_1 d0, d1, s0, s1, s2, s3, s4, s5, s6, s7, s8 vmul.s16 \d0, \s0, d0[0] vmla.s16 \d0, \s1, d0[1] vmla.s16 \d0, \s2, d0[2] vmla.s16 \d0, \s3, d0[3] vmla.s16 \d0, \s4, d1[0] vmla.s16 \d0, \s5, d1[1] vmla.s16 \d0, \s6, d1[2] vmla.s16 \d0, \s7, d1[3] vmul.s16 \d1, \s1, d0[0] vmla.s16 \d1, \s2, d0[1] vmla.s16 \d1, \s3, d0[2] vmla.s16 \d1, \s4, d0[3] vmla.s16 \d1, \s5, d1[0] vmla.s16 \d1, \s6, d1[1] vmla.s16 \d1, \s7, d1[2] vmla.s16 \d1, \s8, d1[3] .endm .macro mul_mla_8_2 d0, d1, s0, s1, s2, s3, s4, s5, s6, s7, s8, s9 vmul.s16 \d0, \s0, d0[0] vmla.s16 \d0, \s1, d0[1] vmla.s16 \d0, \s2, d0[2] vmla.s16 \d0, \s3, d0[3] vmla.s16 \d0, \s4, d1[0] vmla.s16 \d0, \s5, d1[1] vmla.s16 \d0, \s6, d1[2] vmla.s16 \d0, \s7, d1[3] vmul.s16 \d1, \s2, d0[0] vmla.s16 \d1, \s3, d0[1] vmla.s16 \d1, \s4, d0[2] vmla.s16 \d1, \s5, d0[3] vmla.s16 \d1, \s6, d1[0] vmla.s16 \d1, \s7, d1[1] vmla.s16 \d1, \s8, d1[2] vmla.s16 \d1, \s9, d1[3] .endm .macro vqrshrun_s16 shift, q0, d0, q1, d1, q2, d2, q3, d3 vqrshrun.s16 \d0, \q0, #\shift .ifnb \q1 vqrshrun.s16 \d1, \q1, #\shift .endif .ifnb \q2 vqrshrun.s16 \d2, \q2, #\shift vqrshrun.s16 \d3, \q3, #\shift .endif .endm .macro vrshr_s16 shift, r0, r1, r2, r3 vrshr.s16 \r0, \r0, #\shift .ifnb \r1 vrshr.s16 \r1, \r1, #\shift .endif .ifnb \r2 vrshr.s16 \r2, \r2, #\shift vrshr.s16 \r3, \r3, #\shift .endif .endm .macro st_16 strd, reg, lanes vst1.16 {\reg[0]}, [r0, :16], \strd vst1.16 {\reg[1]}, [r8, :16], \strd .if \lanes > 2 vst1.16 {\reg[2]}, [r0, :16], \strd vst1.16 {\reg[3]}, [r8, :16], \strd .endif .endm .macro st_32 strd, r0, r1 vst1.32 {\r0[0]}, [r0, :32], \strd vst1.32 {\r0[1]}, [r8, :32], \strd .ifnb \r1 vst1.32 {\r1[0]}, [r0, :32], \strd vst1.32 {\r1[1]}, [r8, :32], \strd .endif .endm .macro st_reg strd, align, r0, r1, r2, r3, r4, r5, r6, r7 vst1.8 {\r0}, [r0, \align], \strd vst1.8 {\r1}, [r8, \align], \strd .ifnb \r2 vst1.8 {\r2}, [r0, \align], \strd vst1.8 {\r3}, [r8, \align], \strd .endif .ifnb \r4 vst1.8 {\r4}, [r0, \align], \strd vst1.8 {\r5}, [r8, \align], \strd vst1.8 {\r6}, [r0, \align], \strd vst1.8 {\r7}, [r8, \align], \strd .endif .endm .macro shift_store_4 type, strd, q0, d0, d1, q1, d2, d3 .ifc \type, put vqrshrun_s16 6, \q0, \d0, \q1, \d2 st_32 \strd, \d0, \d2 .else vrshr_s16 2, \q0, \q1 st_reg \strd, :64, \d0, \d1, \d2, \d3 .endif .endm .macro shift_store_8 type, strd, q0, d0, q1, d1, q2, d2, q3, d3 .ifc \type, put vqrshrun_s16 6, \q0, \d0, \q1, \d1, \q2, \d2, \q3, \d3 st_reg \strd, :64, \d0, \d1, \d2, \d3 .else vrshr_s16 2, \q0, \q1, \q2, \q3 st_reg \strd, :128,\q0, \q1, \q2, \q3 .endif .endm .macro shift_store_16 type, strd, q0, d0, d1, q1, q2, d4, d5, q3 .ifc \type, put vqrshrun.s16 \d0, \q0, #6 vqrshrun.s16 \d1, \q1, #6 vqrshrun.s16 \d4, \q2, #6 vqrshrun.s16 \d5, \q3, #6 st_reg \strd, :128, \q0, \q2 .else vrshr_s16 2, \q0, \q1, \q2, \q3 vst1.16 {\q0, \q1}, [r0, :128], \strd vst1.16 {\q2, \q3}, [r8, :128], \strd .endif .endm .macro make_8tap_fn op, type, type_h, type_v function \op\()_8tap_\type\()_8bpc_neon, export=1 push {r4-r11,lr} movw r8, \type_h movw r9, \type_v b \op\()_8tap_neon endfunc .endm // No spaces in these expressions, due to gas-preprocessor. #define REGULAR ((0*15<<7)|3*15) #define SMOOTH ((1*15<<7)|4*15) #define SHARP ((2*15<<7)|3*15) .macro filter_fn type, dst, d_strd, src, s_strd, w, h, mx, my, ds2, sr2, shift_hv make_8tap_fn \type, regular, REGULAR, REGULAR make_8tap_fn \type, regular_smooth, REGULAR, SMOOTH make_8tap_fn \type, regular_sharp, REGULAR, SHARP make_8tap_fn \type, smooth, SMOOTH, SMOOTH make_8tap_fn \type, smooth_regular, SMOOTH, REGULAR make_8tap_fn \type, smooth_sharp, SMOOTH, SHARP make_8tap_fn \type, sharp, SHARP, SHARP make_8tap_fn \type, sharp_regular, SHARP, REGULAR make_8tap_fn \type, sharp_smooth, SHARP, SMOOTH function \type\()_8tap_neon ldrd r4, r5, [sp, #36] ldrd r6, r7, [sp, #44] movw r10, #0x4081 // (1 << 14) | (1 << 7) | (1 << 0) mul \mx, \mx, r10 mul \my, \my, r10 add \mx, \mx, r8 // mx, 8tap_h, 4tap_h add \my, \my, r9 // my, 8tap_v, 4tap_v .ifc \type, prep lsl \d_strd, \w, #1 .endif clz r8, \w tst \mx, #(0x7f << 14) sub r8, r8, #24 movrel r10, X(mc_subpel_filters), -8 bne L(\type\()_8tap_h) tst \my, #(0x7f << 14) bne L(\type\()_8tap_v) b \type\()_neon L(\type\()_8tap_h): cmp \w, #4 ubfx r9, \mx, #7, #7 and \mx, \mx, #0x7f it gt movgt \mx, r9 tst \my, #(0x7f << 14) add \mx, r10, \mx, lsl #3 bne L(\type\()_8tap_hv) adr r9, L(\type\()_8tap_h_tbl) ldr r8, [r9, r8, lsl #2] add r9, r9, r8 bx r9 .align 2 L(\type\()_8tap_h_tbl): .word 1280f - L(\type\()_8tap_h_tbl) + CONFIG_THUMB .word 640f - L(\type\()_8tap_h_tbl) + CONFIG_THUMB .word 320f - L(\type\()_8tap_h_tbl) + CONFIG_THUMB .word 160f - L(\type\()_8tap_h_tbl) + CONFIG_THUMB .word 80f - L(\type\()_8tap_h_tbl) + CONFIG_THUMB .word 40f - L(\type\()_8tap_h_tbl) + CONFIG_THUMB .word 20f - L(\type\()_8tap_h_tbl) + CONFIG_THUMB 20: // 2xN h .ifc \type, put add \mx, \mx, #2 vld1.32 {d0[]}, [\mx] sub \src, \src, #1 add \ds2, \dst, \d_strd add \sr2, \src, \s_strd lsl \d_strd, \d_strd, #1 lsl \s_strd, \s_strd, #1 vmovl.s8 q0, d0 2: vld1.8 {d4}, [\src], \s_strd vld1.8 {d6}, [\sr2], \s_strd vmovl.u8 q2, d4 vmovl.u8 q3, d6 vext.8 d5, d4, d5, #2 vext.8 d7, d6, d7, #2 subs \h, \h, #2 vtrn.32 d4, d6 vtrn.32 d5, d7 vmul.s16 d2, d4, d0[0] vmla.s16 d2, d5, d0[1] vmla.s16 d2, d6, d0[2] vmla.s16 d2, d7, d0[3] vrshr.s16 d2, d2, #2 vqrshrun.s16 d2, q1, #4 vst1.16 {d2[0]}, [\dst, :16], \d_strd vst1.16 {d2[1]}, [\ds2, :16], \d_strd bgt 2b pop {r4-r11,pc} .endif 40: // 4xN h add \mx, \mx, #2 vld1.32 {d0[]}, [\mx] sub \src, \src, #1 add \ds2, \dst, \d_strd add \sr2, \src, \s_strd lsl \d_strd, \d_strd, #1 lsl \s_strd, \s_strd, #1 vmovl.s8 q0, d0 4: vld1.8 {d16}, [\src], \s_strd vld1.8 {d24}, [\sr2], \s_strd vmovl.u8 q8, d16 vmovl.u8 q12, d24 vext.8 d18, d16, d17, #2 vext.8 d20, d16, d17, #4 vext.8 d22, d16, d17, #6 vext.8 d26, d24, d25, #2 vext.8 d28, d24, d25, #4 vext.8 d30, d24, d25, #6 subs \h, \h, #2 vmul.s16 d4, d16, d0[0] vmla.s16 d4, d18, d0[1] vmla.s16 d4, d20, d0[2] vmla.s16 d4, d22, d0[3] vmul.s16 d5, d24, d0[0] vmla.s16 d5, d26, d0[1] vmla.s16 d5, d28, d0[2] vmla.s16 d5, d30, d0[3] vrshr.s16 q2, q2, #2 .ifc \type, put vqrshrun.s16 d4, q2, #4 vst1.32 {d4[0]}, [\dst, :32], \d_strd vst1.32 {d4[1]}, [\ds2, :32], \d_strd .else vst1.16 {d4}, [\dst, :64], \d_strd vst1.16 {d5}, [\ds2, :64], \d_strd .endif bgt 4b pop {r4-r11,pc} 80: // 8xN h vld1.8 {d0}, [\mx, :64] sub \src, \src, #3 add \ds2, \dst, \d_strd add \sr2, \src, \s_strd lsl \d_strd, \d_strd, #1 lsl \s_strd, \s_strd, #1 vmovl.s8 q0, d0 8: vld1.8 {q8}, [\src], \s_strd vld1.8 {q12}, [\sr2], \s_strd vmovl.u8 q9, d17 vmovl.u8 q8, d16 vmovl.u8 q13, d25 vmovl.u8 q12, d24 vmul.s16 q10, q8, d0[0] vmul.s16 q14, q12, d0[0] .irpc i, 1234567 vext.8 q11, q8, q9, #(2*\i) vext.8 q15, q12, q13, #(2*\i) .if \i < 4 vmla.s16 q10, q11, d0[\i] vmla.s16 q14, q15, d0[\i] .else vmla.s16 q10, q11, d1[\i-4] vmla.s16 q14, q15, d1[\i-4] .endif .endr subs \h, \h, #2 vrshr.s16 q10, q10, #2 vrshr.s16 q14, q14, #2 .ifc \type, put vqrshrun.s16 d20, q10, #4 vqrshrun.s16 d28, q14, #4 vst1.8 {d20}, [\dst, :64], \d_strd vst1.8 {d28}, [\ds2, :64], \d_strd .else vst1.16 {q10}, [\dst, :128], \d_strd vst1.16 {q14}, [\ds2, :128], \d_strd .endif bgt 8b pop {r4-r11,pc} 160: 320: 640: 1280: // 16xN, 32xN, ... h // This could be done without touching q4-q6, by using only // one temporary for vext in the loop. That's slower on A7 and A53, // (but surprisingly, marginally faster on A8 and A73). vpush {q4-q6} vld1.8 {d0}, [\mx, :64] sub \src, \src, #3 add \ds2, \dst, \d_strd add \sr2, \src, \s_strd lsl \s_strd, \s_strd, #1 vmovl.s8 q0, d0 sub \s_strd, \s_strd, \w sub \s_strd, \s_strd, #8 .ifc \type, put lsl \d_strd, \d_strd, #1 sub \d_strd, \d_strd, \w .endif 161: vld1.8 {d16, d17, d18}, [\src]! vld1.8 {d24, d25, d26}, [\sr2]! mov \mx, \w vmovl.u8 q10, d18 vmovl.u8 q9, d17 vmovl.u8 q8, d16 vmovl.u8 q14, d26 vmovl.u8 q13, d25 vmovl.u8 q12, d24 16: vmul.s16 q1, q8, d0[0] vmul.s16 q2, q9, d0[0] vmul.s16 q3, q12, d0[0] vmul.s16 q4, q13, d0[0] .irpc i, 1234567 vext.8 q5, q8, q9, #(2*\i) vext.8 q6, q9, q10, #(2*\i) vext.8 q11, q12, q13, #(2*\i) vext.8 q15, q13, q14, #(2*\i) .if \i < 4 vmla.s16 q1, q5, d0[\i] vmla.s16 q2, q6, d0[\i] vmla.s16 q3, q11, d0[\i] vmla.s16 q4, q15, d0[\i] .else vmla.s16 q1, q5, d1[\i-4] vmla.s16 q2, q6, d1[\i-4] vmla.s16 q3, q11, d1[\i-4] vmla.s16 q4, q15, d1[\i-4] .endif .endr vrshr.s16 q1, q1, #2 vrshr.s16 q2, q2, #2 vrshr.s16 q3, q3, #2 vrshr.s16 q4, q4, #2 subs \mx, \mx, #16 .ifc \type, put vqrshrun.s16 d2, q1, #4 vqrshrun.s16 d3, q2, #4 vqrshrun.s16 d4, q3, #4 vqrshrun.s16 d5, q4, #4 vst1.8 {q1}, [\dst, :128]! vst1.8 {q2}, [\ds2, :128]! .else vst1.16 {q1, q2}, [\dst, :128]! vst1.16 {q3, q4}, [\ds2, :128]! .endif ble 9f vmov q8, q10 vmov q12, q14 vld1.8 {d18, d19}, [\src]! vld1.8 {d26, d27}, [\sr2]! vmovl.u8 q10, d19 vmovl.u8 q9, d18 vmovl.u8 q14, d27 vmovl.u8 q13, d26 b 16b 9: add \dst, \dst, \d_strd add \ds2, \ds2, \d_strd add \src, \src, \s_strd add \sr2, \sr2, \s_strd subs \h, \h, #2 bgt 161b vpop {q4-q6} pop {r4-r11,pc} L(\type\()_8tap_v): cmp \h, #4 ubfx r9, \my, #7, #7 and \my, \my, #0x7f it gt movgt \my, r9 add \my, r10, \my, lsl #3 adr r9, L(\type\()_8tap_v_tbl) ldr r8, [r9, r8, lsl #2] add r9, r9, r8 bx r9 .align 2 L(\type\()_8tap_v_tbl): .word 1280f - L(\type\()_8tap_v_tbl) + CONFIG_THUMB .word 640f - L(\type\()_8tap_v_tbl) + CONFIG_THUMB .word 320f - L(\type\()_8tap_v_tbl) + CONFIG_THUMB .word 160f - L(\type\()_8tap_v_tbl) + CONFIG_THUMB .word 80f - L(\type\()_8tap_v_tbl) + CONFIG_THUMB .word 40f - L(\type\()_8tap_v_tbl) + CONFIG_THUMB .word 20f - L(\type\()_8tap_v_tbl) + CONFIG_THUMB 20: // 2xN v .ifc \type, put bgt 28f cmp \h, #2 add \my, \my, #2 vld1.32 {d0[]}, [\my] sub \src, \src, \s_strd add \ds2, \dst, \d_strd add \sr2, \src, \s_strd lsl \s_strd, \s_strd, #1 lsl \d_strd, \d_strd, #1 vmovl.s8 q0, d0 // 2x2 v load_16 \src, \sr2, \s_strd, d1, d2, d3, d4, d5 interleave_1_16 d1, d2, d3, d4, d5 bgt 24f vmovl_u8 q8, d1, q9, d2, q10, d3, q11, d4 mul_mla_4 d6, d16, d18, d20, d22 vqrshrun_s16 6, q3, d6 st_16 \d_strd, d6, 2 pop {r4-r11,pc} 24: // 2x4 v load_16 \sr2, \src, \s_strd, d6, d7 interleave_1_16 d5, d6, d7 vmovl_u8 q8, d1, q9, d2, q10, d3, q11, d4, q12, d5, q13, d6 vmov d17, d20 vmov d19, d22 vmov d21, d24 vmov d23, d26 mul_mla_4 q3, q8, q9, q10, q11 vqrshrun_s16 6, q3, d6 st_16 \d_strd, d6, 4 pop {r4-r11,pc} 28: // 2x6, 2x8, 2x12, 2x16 v vpush {q4-q7} vld1.8 {d0}, [\my, :64] sub \sr2, \src, \s_strd, lsl #1 add \ds2, \dst, \d_strd sub \src, \sr2, \s_strd lsl \d_strd, \d_strd, #1 lsl \s_strd, \s_strd, #1 vmovl.s8 q0, d0 load_16 \src, \sr2, \s_strd, d2, d4, d6, d8, d10, d12, d14 interleave_1_16 d2, d4, d6, d8, d10 interleave_1_16 d10, d12, d14 vmovl_u8 q1, d2, q2, d4, q3, d6, q4, d8, q5, d10, q6, d12 vmov d3, d6 vmov d5, d8 vmov d7, d10 vmov d9, d12 216: subs \h, \h, #4 load_16 \sr2, \src, \s_strd, d16, d18, d20, d22 interleave_1_16 d14, d16, d18, d20, d22 vmovl_u8 q7, d14, q8, d16, q9, d18, q10, d20 vmov d11, d14 vmov d13, d16 vmov d15, d18 vmov d17, d20 mul_mla_8_0 q1, q1, q2, q3, q4, q5, q6, q7, q8 vqrshrun_s16 6, q1, d2 st_16 \d_strd, d2, 4 ble 0f cmp \h, #2 vmov q1, q5 vmov q2, q6 vmov q3, q7 vmov q4, q8 vmov q5, q9 vmov q6, q10 vmov d14, d22 beq 26f b 216b 26: load_16 \sr2, \src, \s_strd, d16, d18 interleave_1_16 d14, d16, d18 vmovl_u8 q7, d14, q8, d16 vmov d11, d14 vmov d13, d16 mul_mla_8_0 d2, d2, d4, d6, d8, d10, d12, d14, d16 vqrshrun_s16 6, q1, d2 st_16 \d_strd, d2, 2 0: vpop {q4-q7} pop {r4-r11,pc} .endif 40: bgt 480f // 4x2, 4x4 v cmp \h, #2 add \my, \my, #2 vld1.32 {d0[]}, [\my] sub \src, \src, \s_strd add \ds2, \dst, \d_strd add \sr2, \src, \s_strd lsl \s_strd, \s_strd, #1 lsl \d_strd, \d_strd, #1 vmovl.s8 q0, d0 load_32 \src, \sr2, \s_strd, d1, d2, d3, d4, d5 interleave_1_32 d1, d2, d3, d4, d5 vmovl_u8 q8, d1, q9, d2, q10, d3, q11, d4 mul_mla_4 q3, q8, q9, q10, q11 shift_store_4 \type, \d_strd, q3, d6, d7 ble 0f load_32 \sr2, \src, \s_strd, d6, d7 interleave_1_32 d5, d6, d7 vmovl_u8 q12, d5, q13, d6 mul_mla_4 q3, q10, q11, q12, q13 shift_store_4 \type, \d_strd, q3, d6, d7 0: pop {r4-r11,pc} 480: // 4x6, 4x8, 4x12, 4x16 v vpush {q4} vld1.8 {d0}, [\my, :64] sub \sr2, \src, \s_strd, lsl #1 add \ds2, \dst, \d_strd sub \src, \sr2, \s_strd lsl \s_strd, \s_strd, #1 lsl \d_strd, \d_strd, #1 vmovl.s8 q0, d0 load_32 \src, \sr2, \s_strd, d2, d4, d6, d8, d16, d18, d20 interleave_1_32 d2, d4, d6 interleave_1_32 d6, d8, d16, d18, d20 vmovl_u8 q1, d2, q2, d4, q3, d6, q4, d8, q8, d16, q9, d18 48: subs \h, \h, #4 load_32 \sr2, \src, \s_strd, d22, d24, d26, d28 interleave_1_32 d20, d22, d24, d26, d28 vmovl_u8 q10, d20, q11, d22, q12, d24, q13, d26 mul_mla_8_2 q1, q2, q1, q2, q3, q4, q8, q9, q10, q11, q12, q13 shift_store_4 \type, \d_strd, q1, d2, d3, q2, d4, d5 ble 0f load_32 \sr2, \src, \s_strd, d30, d2 subs \h, \h, #2 interleave_1_32 d28, d30, d2 vmovl_u8 q14, d28, q15, d30 mul_mla_8_0 q8, q8, q9, q10, q11, q12, q13, q14, q15 shift_store_4 \type, \d_strd, q8, d16, d17 ble 0f load_32 \sr2, \src, \s_strd, d4, d6 subs \h, \h, #2 interleave_1_32 d2, d4, d6 vmovl_u8 q1, d2, q2, d4 mul_mla_8_0 q9, q10, q11, q12, q13, q14, q15, q1, q2 shift_store_4 \type, \d_strd, q9, d18, d19 ble 0f subs \h, \h, #4 load_32 \sr2, \src, \s_strd, d8, d16, d18, d20 interleave_1_32 d6, d8, d16, d18, d20 vmovl_u8 q3, d6, q4, d8, q8, d16, q9, d18 mul_mla_8_2 q12, q13, q12, q13, q14, q15, q1, q2, q3, q4, q8, q9 shift_store_4 \type, \d_strd, q12, d24, d25, q13, d26, d27 bgt 48b 0: vpop {q4} pop {r4-r11,pc} 80: bgt 880f // 8x2, 8x4 v cmp \h, #2 add \my, \my, #2 vld1.32 {d0[]}, [\my] sub \src, \src, \s_strd add \ds2, \dst, \d_strd add \sr2, \src, \s_strd lsl \s_strd, \s_strd, #1 lsl \d_strd, \d_strd, #1 vmovl.s8 q0, d0 load_reg \src, \sr2, \s_strd, d1, d2, d3, d4, d5 vmovl_u8 q8, d1, q9, d2, q10, d3, q11, d4, q12, d5 mul_mla_4 q1, q8, q9, q10, q11 mul_mla_4 q2, q9, q10, q11, q12 shift_store_8 \type, \d_strd, q1, d2, q2, d4 ble 0f load_reg \sr2, \src, \s_strd, d6, d7 vmovl_u8 q13, d6, q14, d7 mul_mla_4 q1, q10, q11, q12, q13 mul_mla_4 q2, q11, q12, q13, q14 shift_store_8 \type, \d_strd, q1, d2, q2, d4 0: pop {r4-r11,pc} 880: // 8x6, 8x8, 8x16, 8x32 v 1680: // 16x8, 16x16, ... 320: // 32x8, 32x16, ... 640: 1280: vpush {q4} vld1.8 {d0}, [\my, :64] sub \src, \src, \s_strd sub \src, \src, \s_strd, lsl #1 vmovl.s8 q0, d0 mov \my, \h 168: add \ds2, \dst, \d_strd add \sr2, \src, \s_strd lsl \s_strd, \s_strd, #1 lsl \d_strd, \d_strd, #1 load_reg \src, \sr2, \s_strd, d2, d4, d6, d8, d16, d18, d20 vmovl_u8 q1, d2, q2, d4, q3, d6, q4, d8, q8, d16, q9, d18, q10, d20 88: subs \h, \h, #2 load_reg \sr2, \src, \s_strd, d22, d24 vmovl_u8 q11, d22, q12, d24 mul_mla_8_1 q1, q2, q1, q2, q3, q4, q8, q9, q10, q11, q12 shift_store_8 \type, \d_strd, q1, d2, q2, d4 ble 9f subs \h, \h, #2 load_reg \sr2, \src, \s_strd, d26, d28 vmovl_u8 q13, d26, q14, d28 mul_mla_8_1 q3, q4, q3, q4, q8, q9, q10, q11, q12, q13, q14 shift_store_8 \type, \d_strd, q3, d6, q4, d8 ble 9f subs \h, \h, #2 load_reg \sr2, \src, \s_strd, d30, d2 vmovl_u8 q15, d30, q1, d2 mul_mla_8_1 q8, q9, q8, q9, q10, q11, q12, q13, q14, q15, q1 shift_store_8 \type, \d_strd, q8, d16, q9, d18 ble 9f subs \h, \h, #2 load_reg \sr2, \src, \s_strd, d4, d6 vmovl_u8 q2, d4, q3, d6 mul_mla_8_1 q10, q11, q10, q11, q12, q13, q14, q15, q1, q2, q3 shift_store_8 \type, \d_strd, q10, d20, q11, d22 ble 9f subs \h, \h, #4 load_reg \sr2, \src, \s_strd, d8, d16, d18, d20 vmovl_u8 q4, d8, q8, d16, q9, d18, q10, d20 mul_mla_8_1 q12, q13, q12, q13, q14, q15, q1, q2, q3, q4, q8 mul_mla_8_1 q14, q15, q14, q15, q1, q2, q3, q4, q8, q9, q10 shift_store_8 \type, \d_strd, q12, d24, q13, d26, q14, d28, q15, d30 bgt 88b 9: subs \w, \w, #8 ble 0f asr \s_strd, \s_strd, #1 asr \d_strd, \d_strd, #1 mls \src, \s_strd, \my, \src mls \dst, \d_strd, \my, \dst sub \src, \src, \s_strd, lsl #3 mov \h, \my add \src, \src, #8 .ifc \type, put add \dst, \dst, #8 .else add \dst, \dst, #16 .endif b 168b 0: vpop {q4} pop {r4-r11,pc} 160: bgt 1680b // 16x2, 16x4 v add \my, \my, #2 vld1.32 {d0[]}, [\my] sub \src, \src, \s_strd add \ds2, \dst, \d_strd add \sr2, \src, \s_strd lsl \s_strd, \s_strd, #1 lsl \d_strd, \d_strd, #1 vmovl.s8 q0, d0 cmp \h, #2 load_reg \src, \sr2, \s_strd, q11, q12, q13, q14, q15 vmovl.u8 q1, d22 vmovl.u8 q2, d24 vmovl.u8 q3, d26 vmovl.u8 q8, d28 vmovl.u8 q9, d30 vmovl.u8 q11, d23 vmovl.u8 q12, d25 vmovl.u8 q13, d27 vmovl.u8 q14, d29 vmovl.u8 q15, d31 mul_mla_4 q1, q1, q2, q3, q8 mul_mla_4 q10, q2, q3, q8, q9 mul_mla_4 q2, q11, q12, q13, q14 mul_mla_4 q11, q12, q13, q14, q15 shift_store_16 \type, \d_strd, q1, d2, d3, q2, q10, d20, d21, q11 ble 0f load_reg \sr2, \src, \s_strd, q10, q11 vmovl.u8 q1, d20 vmovl.u8 q10, d21 vmovl.u8 q12, d22 vmovl.u8 q11, d23 mul_mla_4 q2, q3, q8, q9, q1 mul_mla_4 q3, q13, q14, q15, q10 mul_mla_4 q13, q8, q9, q1, q12 mul_mla_4 q14, q14, q15, q10, q11 shift_store_16 \type, \d_strd, q2, d4, d5, q3, q13, d26, d27, q14 0: pop {r4-r11,pc} L(\type\()_8tap_hv): cmp \h, #4 ubfx r9, \my, #7, #7 and \my, \my, #0x7f it gt movgt \my, r9 add \my, r10, \my, lsl #3 adr r9, L(\type\()_8tap_hv_tbl) ldr r8, [r9, r8, lsl #2] add r9, r9, r8 bx r9 .align 2 L(\type\()_8tap_hv_tbl): .word 1280f - L(\type\()_8tap_hv_tbl) + CONFIG_THUMB .word 640f - L(\type\()_8tap_hv_tbl) + CONFIG_THUMB .word 320f - L(\type\()_8tap_hv_tbl) + CONFIG_THUMB .word 160f - L(\type\()_8tap_hv_tbl) + CONFIG_THUMB .word 80f - L(\type\()_8tap_hv_tbl) + CONFIG_THUMB .word 40f - L(\type\()_8tap_hv_tbl) + CONFIG_THUMB .word 20f - L(\type\()_8tap_hv_tbl) + CONFIG_THUMB 20: .ifc \type, put add \mx, \mx, #2 vld1.32 {d0[]}, [\mx] bgt 280f add \my, \my, #2 vld1.32 {d2[]}, [\my] // 2x2, 2x4 hv sub \sr2, \src, #1 sub \src, \sr2, \s_strd add \ds2, \dst, \d_strd lsl \s_strd, \s_strd, #1 lsl \d_strd, \d_strd, #1 vmovl.s8 q0, d0 vmovl.s8 q1, d2 vld1.8 {d26}, [\src], \s_strd vmovl.u8 q13, d26 vext.8 q14, q13, q13, #2 vmul.s16 d26, d26, d0 vmul.s16 d28, d28, d0 vpadd.s16 d26, d26, d28 vpadd.s16 d26, d26, d26 vrshr.s16 d16, d26, #2 bl L(\type\()_8tap_filter_2) vext.8 d16, d16, d16, #4 vmov d17, d26 vext.8 d16, d16, d26, #4 2: bl L(\type\()_8tap_filter_2) vext.8 d18, d17, d26, #4 vmull.s16 q2, d16, d2[0] vmlal.s16 q2, d17, d2[1] vmlal.s16 q2, d18, d2[2] vmlal.s16 q2, d26, d2[3] vqrshrn.s32 d4, q2, #\shift_hv vqmovun.s16 d4, q2 subs \h, \h, #2 vst1.16 {d4[0]}, [\dst, :16], \d_strd vst1.16 {d4[1]}, [\ds2, :16], \d_strd ble 0f vmov d16, d18 vmov d17, d26 b 2b 280: // 2x8, 2x16, 2x32 hv vld1.8 {d2}, [\my, :64] sub \src, \src, #1 sub \sr2, \src, \s_strd, lsl #1 sub \src, \sr2, \s_strd add \ds2, \dst, \d_strd lsl \s_strd, \s_strd, #1 lsl \d_strd, \d_strd, #1 vmovl.s8 q0, d0 vmovl.s8 q1, d2 vld1.8 {d26}, [\src], \s_strd vmovl.u8 q13, d26 vext.8 q14, q13, q13, #2 vmul.s16 d26, d26, d0 vmul.s16 d28, d28, d0 vpadd.s16 d26, d26, d28 vpadd.s16 d26, d26, d26 vrshr.s16 d16, d26, #2 bl L(\type\()_8tap_filter_2) vext.8 d16, d16, d16, #4 vmov d17, d26 vext.8 d16, d16, d26, #4 bl L(\type\()_8tap_filter_2) vext.8 d18, d17, d26, #4 vmov d19, d26 bl L(\type\()_8tap_filter_2) vext.8 d20, d19, d26, #4 vmov d21, d26 28: bl L(\type\()_8tap_filter_2) vext.8 d22, d21, d26, #4 vmull.s16 q2, d16, d2[0] vmlal.s16 q2, d17, d2[1] vmlal.s16 q2, d18, d2[2] vmlal.s16 q2, d19, d2[3] vmlal.s16 q2, d20, d3[0] vmlal.s16 q2, d21, d3[1] vmlal.s16 q2, d22, d3[2] vmlal.s16 q2, d26, d3[3] vqrshrn.s32 d4, q2, #\shift_hv vqmovun.s16 d4, q2 subs \h, \h, #2 vst1.16 {d4[0]}, [\dst, :16], \d_strd vst1.16 {d4[1]}, [\ds2, :16], \d_strd ble 0f vmov d16, d18 vmov d17, d19 vmov d18, d20 vmov d19, d21 vmov d20, d22 vmov d21, d26 b 28b 0: pop {r4-r11,pc} L(\type\()_8tap_filter_2): vld1.8 {d28}, [\sr2], \s_strd vld1.8 {d30}, [\src], \s_strd vext.8 d29, d28, d28, #1 vext.8 d31, d30, d30, #1 vmovl.u8 q13, d28 vmovl.u8 q14, d29 vmov d27, d28 vmovl.u8 q14, d30 vmovl.u8 q15, d31 vtrn.32 d26, d28 vtrn.32 d27, d30 vmul.s16 d26, d26, d0[0] vmla.s16 d26, d27, d0[1] vmla.s16 d26, d28, d0[2] vmla.s16 d26, d30, d0[3] vrshr.s16 d26, d26, #2 vext.8 d27, d26, d26, #4 bx lr .endif 40: add \mx, \mx, #2 vld1.32 {d0[]}, [\mx] bgt 480f add \my, \my, #2 vld1.32 {d2[]}, [\my] sub \sr2, \src, #1 sub \src, \sr2, \s_strd add \ds2, \dst, \d_strd lsl \s_strd, \s_strd, #1 lsl \d_strd, \d_strd, #1 vmovl.s8 q0, d0 vmovl.s8 q1, d2 // 4x2, 4x4 hv vld1.8 {d30}, [\src], \s_strd vmovl.u8 q14, d30 vext.8 d27, d28, d29, #2 vext.8 d30, d28, d29, #4 vext.8 d31, d28, d29, #6 vmul.s16 d26, d28, d0[0] vmla.s16 d26, d27, d0[1] vmla.s16 d26, d30, d0[2] vmla.s16 d26, d31, d0[3] vrshr.s16 d16, d26, #2 bl L(\type\()_8tap_filter_4) vmov d17, d26 vmov d18, d27 4: bl L(\type\()_8tap_filter_4) vmull.s16 q2, d16, d2[0] vmlal.s16 q2, d17, d2[1] vmlal.s16 q2, d18, d2[2] vmlal.s16 q2, d26, d2[3] vmull.s16 q3, d17, d2[0] vmlal.s16 q3, d18, d2[1] vmlal.s16 q3, d26, d2[2] vmlal.s16 q3, d27, d2[3] vqrshrn.s32 d4, q2, #\shift_hv vqrshrn.s32 d6, q3, #\shift_hv subs \h, \h, #2 .ifc \type, put vqmovun.s16 d4, q2 vqmovun.s16 d6, q3 vst1.32 {d4[0]}, [\dst, :32], \d_strd vst1.32 {d6[0]}, [\ds2, :32], \d_strd .else vst1.16 {d4}, [\dst, :64], \d_strd vst1.16 {d6}, [\ds2, :64], \d_strd .endif ble 0f vmov d16, d18 vmov d17, d26 vmov d18, d27 b 4b 480: // 4x8, 4x16, 4x32 hv vld1.8 {d2}, [\my, :64] sub \src, \src, #1 sub \sr2, \src, \s_strd, lsl #1 sub \src, \sr2, \s_strd add \ds2, \dst, \d_strd lsl \s_strd, \s_strd, #1 lsl \d_strd, \d_strd, #1 vmovl.s8 q0, d0 vmovl.s8 q1, d2 vld1.8 {d30}, [\src], \s_strd vmovl.u8 q14, d30 vext.8 d27, d28, d29, #2 vext.8 d30, d28, d29, #4 vext.8 d31, d28, d29, #6 vmul.s16 d26, d28, d0[0] vmla.s16 d26, d27, d0[1] vmla.s16 d26, d30, d0[2] vmla.s16 d26, d31, d0[3] vrshr.s16 d16, d26, #2 bl L(\type\()_8tap_filter_4) vmov d17, d26 vmov d18, d27 bl L(\type\()_8tap_filter_4) vmov d19, d26 vmov d20, d27 bl L(\type\()_8tap_filter_4) vmov d21, d26 vmov d22, d27 48: bl L(\type\()_8tap_filter_4) vmull.s16 q2, d16, d2[0] vmlal.s16 q2, d17, d2[1] vmlal.s16 q2, d18, d2[2] vmlal.s16 q2, d19, d2[3] vmlal.s16 q2, d20, d3[0] vmlal.s16 q2, d21, d3[1] vmlal.s16 q2, d22, d3[2] vmlal.s16 q2, d26, d3[3] vmull.s16 q3, d17, d2[0] vmlal.s16 q3, d18, d2[1] vmlal.s16 q3, d19, d2[2] vmlal.s16 q3, d20, d2[3] vmlal.s16 q3, d21, d3[0] vmlal.s16 q3, d22, d3[1] vmlal.s16 q3, d26, d3[2] vmlal.s16 q3, d27, d3[3] vqrshrn.s32 d4, q2, #\shift_hv vqrshrn.s32 d6, q3, #\shift_hv subs \h, \h, #2 .ifc \type, put vqmovun.s16 d4, q2 vqmovun.s16 d6, q3 vst1.32 {d4[0]}, [\dst, :32], \d_strd vst1.32 {d6[0]}, [\ds2, :32], \d_strd .else vst1.16 {d4}, [\dst, :64], \d_strd vst1.16 {d6}, [\ds2, :64], \d_strd .endif ble 0f vmov d16, d18 vmov d17, d19 vmov d18, d20 vmov d19, d21 vmov d20, d22 vmov d21, d26 vmov d22, d27 b 48b 0: pop {r4-r11,pc} L(\type\()_8tap_filter_4): vld1.8 {d30}, [\sr2], \s_strd vld1.8 {d31}, [\src], \s_strd vmovl.u8 q14, d30 vext.8 d27, d28, d29, #2 vext.8 d30, d28, d29, #4 vext.8 d1, d28, d29, #6 vmul.s16 d26, d28, d0[0] vmla.s16 d26, d27, d0[1] vmla.s16 d26, d30, d0[2] vmla.s16 d26, d1, d0[3] vmovl.u8 q14, d31 vext.8 d30, d28, d29, #2 vext.8 d31, d28, d29, #4 vext.8 d1, d28, d29, #6 vmul.s16 d27, d28, d0[0] vmla.s16 d27, d30, d0[1] vmla.s16 d27, d31, d0[2] vmla.s16 d27, d1, d0[3] vrshr.s16 d26, d26, #2 vrshr.s16 d27, d27, #2 bx lr 80: 160: 320: bgt 880f vpush {q4-q7} add \my, \my, #2 vld1.8 {d0}, [\mx, :64] vld1.32 {d2[]}, [\my] sub \src, \src, #3 sub \src, \src, \s_strd vmovl.s8 q0, d0 vmovl.s8 q1, d2 mov \my, \h 164: // 8x2, 8x4, 16x2, 16x4, 32x2, 32x4 hv add \ds2, \dst, \d_strd add \sr2, \src, \s_strd lsl \d_strd, \d_strd, #1 lsl \s_strd, \s_strd, #1 vld1.8 {q14}, [\src], \s_strd vmovl.u8 q12, d28 vmovl.u8 q13, d29 vmul.s16 q10, q12, d0[0] .irpc i, 123 vext.8 q14, q12, q13, #(2*\i) vmla.s16 q10, q14, d0[\i] .endr .irpc i, 4567 vext.8 q14, q12, q13, #(2*\i) vmla.s16 q10, q14, d1[\i-4] .endr vrshr.s16 q3, q10, #2 bl L(\type\()_8tap_filter_8) vmov q4, q10 vmov q5, q11 8: bl L(\type\()_8tap_filter_8) vmull.s16 q12, d6, d2[0] vmull.s16 q13, d7, d2[0] vmull.s16 q14, d8, d2[0] vmull.s16 q15, d9, d2[0] vmlal.s16 q12, d8, d2[1] vmlal.s16 q13, d9, d2[1] vmlal.s16 q14, d10, d2[1] vmlal.s16 q15, d11, d2[1] vmlal.s16 q12, d10, d2[2] vmlal.s16 q13, d11, d2[2] vmlal.s16 q14, d20, d2[2] vmlal.s16 q15, d21, d2[2] vmlal.s16 q12, d20, d2[3] vmlal.s16 q13, d21, d2[3] vmlal.s16 q14, d22, d2[3] vmlal.s16 q15, d23, d2[3] vqrshrn.s32 d24, q12, #\shift_hv vqrshrn.s32 d25, q13, #\shift_hv vqrshrn.s32 d28, q14, #\shift_hv vqrshrn.s32 d29, q15, #\shift_hv subs \h, \h, #2 .ifc \type, put vqmovun.s16 d24, q12 vqmovun.s16 d28, q14 vst1.8 {d24}, [\dst, :64], \d_strd vst1.8 {d28}, [\ds2, :64], \d_strd .else vst1.16 {q12}, [\dst, :128], \d_strd vst1.16 {q14}, [\ds2, :128], \d_strd .endif ble 9f vmov q3, q5 vmov q4, q10 vmov q5, q11 b 8b 9: subs \w, \w, #8 ble 0f asr \s_strd, \s_strd, #1 asr \d_strd, \d_strd, #1 mls \src, \s_strd, \my, \src mls \dst, \d_strd, \my, \dst sub \src, \src, \s_strd, lsl #2 mov \h, \my add \src, \src, #8 .ifc \type, put add \dst, \dst, #8 .else add \dst, \dst, #16 .endif b 164b 880: // 8x8, 8x16, ..., 16x8, ..., 32x8, ... hv 640: 1280: vpush {q4-q7} vld1.8 {d0}, [\mx, :64] vld1.8 {d2}, [\my, :64] sub \src, \src, #3 sub \src, \src, \s_strd sub \src, \src, \s_strd, lsl #1 vmovl.s8 q0, d0 vmovl.s8 q1, d2 mov \my, \h 168: add \ds2, \dst, \d_strd add \sr2, \src, \s_strd lsl \d_strd, \d_strd, #1 lsl \s_strd, \s_strd, #1 vld1.8 {q14}, [\src], \s_strd vmovl.u8 q12, d28 vmovl.u8 q13, d29 vmul.s16 q10, q12, d0[0] .irpc i, 123 vext.8 q14, q12, q13, #(2*\i) vmla.s16 q10, q14, d0[\i] .endr .irpc i, 4567 vext.8 q14, q12, q13, #(2*\i) vmla.s16 q10, q14, d1[\i-4] .endr vrshr.s16 q3, q10, #2 bl L(\type\()_8tap_filter_8) vmov q4, q10 vmov q5, q11 bl L(\type\()_8tap_filter_8) vmov q6, q10 vmov q7, q11 bl L(\type\()_8tap_filter_8) vmov q8, q10 vmov q9, q11 88: bl L(\type\()_8tap_filter_8) vmull.s16 q12, d6, d2[0] vmull.s16 q13, d7, d2[0] vmull.s16 q14, d8, d2[0] vmull.s16 q15, d9, d2[0] vmlal.s16 q12, d8, d2[1] vmlal.s16 q13, d9, d2[1] vmlal.s16 q14, d10, d2[1] vmlal.s16 q15, d11, d2[1] vmlal.s16 q12, d10, d2[2] vmlal.s16 q13, d11, d2[2] vmlal.s16 q14, d12, d2[2] vmlal.s16 q15, d13, d2[2] vmlal.s16 q12, d12, d2[3] vmlal.s16 q13, d13, d2[3] vmlal.s16 q14, d14, d2[3] vmlal.s16 q15, d15, d2[3] vmlal.s16 q12, d14, d3[0] vmlal.s16 q13, d15, d3[0] vmlal.s16 q14, d16, d3[0] vmlal.s16 q15, d17, d3[0] vmlal.s16 q12, d16, d3[1] vmlal.s16 q13, d17, d3[1] vmlal.s16 q14, d18, d3[1] vmlal.s16 q15, d19, d3[1] vmlal.s16 q12, d18, d3[2] vmlal.s16 q13, d19, d3[2] vmlal.s16 q14, d20, d3[2] vmlal.s16 q15, d21, d3[2] vmlal.s16 q12, d20, d3[3] vmlal.s16 q13, d21, d3[3] vmlal.s16 q14, d22, d3[3] vmlal.s16 q15, d23, d3[3] vqrshrn.s32 d24, q12, #\shift_hv vqrshrn.s32 d25, q13, #\shift_hv vqrshrn.s32 d28, q14, #\shift_hv vqrshrn.s32 d29, q15, #\shift_hv subs \h, \h, #2 .ifc \type, put vqmovun.s16 d24, q12 vqmovun.s16 d28, q14 vst1.8 {d24}, [\dst, :64], \d_strd vst1.8 {d28}, [\ds2, :64], \d_strd .else vst1.16 {q12}, [\dst, :128], \d_strd vst1.16 {q14}, [\ds2, :128], \d_strd .endif ble 9f vmov q3, q5 vmov q4, q6 vmov q5, q7 vmov q6, q8 vmov q7, q9 vmov q8, q10 vmov q9, q11 b 88b 9: subs \w, \w, #8 ble 0f asr \s_strd, \s_strd, #1 asr \d_strd, \d_strd, #1 mls \src, \s_strd, \my, \src mls \dst, \d_strd, \my, \dst sub \src, \src, \s_strd, lsl #3 mov \h, \my add \src, \src, #8 .ifc \type, put add \dst, \dst, #8 .else add \dst, \dst, #16 .endif b 168b 0: vpop {q4-q7} pop {r4-r11,pc} L(\type\()_8tap_filter_8): vld1.8 {q14}, [\sr2], \s_strd vld1.8 {q15}, [\src], \s_strd vmovl.u8 q12, d28 vmovl.u8 q13, d29 vmul.s16 q10, q12, d0[0] .irpc i, 123 vext.8 q14, q12, q13, #(2*\i) vmla.s16 q10, q14, d0[\i] .endr .irpc i, 4567 vext.8 q14, q12, q13, #(2*\i) vmla.s16 q10, q14, d1[\i-4] .endr vmovl.u8 q12, d30 vmovl.u8 q13, d31 vmul.s16 q11, q12, d0[0] .irpc i, 123 vext.8 q14, q12, q13, #(2*\i) vmla.s16 q11, q14, d0[\i] .endr .irpc i, 4567 vext.8 q14, q12, q13, #(2*\i) vmla.s16 q11, q14, d1[\i-4] .endr vrshr.s16 q10, q10, #2 vrshr.s16 q11, q11, #2 bx lr endfunc function \type\()_bilin_8bpc_neon, export=1 push {r4-r11,lr} ldrd r4, r5, [sp, #36] ldrd r6, r7, [sp, #44] vdup.8 d1, \mx vdup.8 d3, \my rsb r8, \mx, #16 rsb r9, \my, #16 vdup.8 d0, r8 vdup.8 d2, r9 .ifc \type, prep lsl \d_strd, \w, #1 .endif clz r8, \w cmp \mx, #0 sub r8, r8, #24 bne L(\type\()_bilin_h) cmp \my, #0 bne L(\type\()_bilin_v) b \type\()_neon L(\type\()_bilin_h): cmp \my, #0 bne L(\type\()_bilin_hv) adr r9, L(\type\()_bilin_h_tbl) ldr r8, [r9, r8, lsl #2] add r9, r9, r8 bx r9 .align 2 L(\type\()_bilin_h_tbl): .word 1280f - L(\type\()_bilin_h_tbl) + CONFIG_THUMB .word 640f - L(\type\()_bilin_h_tbl) + CONFIG_THUMB .word 320f - L(\type\()_bilin_h_tbl) + CONFIG_THUMB .word 160f - L(\type\()_bilin_h_tbl) + CONFIG_THUMB .word 80f - L(\type\()_bilin_h_tbl) + CONFIG_THUMB .word 40f - L(\type\()_bilin_h_tbl) + CONFIG_THUMB .word 20f - L(\type\()_bilin_h_tbl) + CONFIG_THUMB 20: // 2xN h .ifc \type, put add \ds2, \dst, \d_strd add \sr2, \src, \s_strd lsl \d_strd, \d_strd, #1 lsl \s_strd, \s_strd, #1 2: vld1.32 {d4[]}, [\src], \s_strd vld1.32 {d6[]}, [\sr2], \s_strd vext.8 d5, d4, d4, #1 vext.8 d7, d6, d6, #1 vtrn.16 q2, q3 subs \h, \h, #2 vmull.u8 q3, d4, d0 vmlal.u8 q3, d5, d1 vqrshrn.u16 d4, q3, #4 vst1.16 {d4[0]}, [\dst, :16], \d_strd vst1.16 {d4[1]}, [\ds2, :16], \d_strd bgt 2b pop {r4-r11,pc} .endif 40: // 4xN h add \ds2, \dst, \d_strd add \sr2, \src, \s_strd lsl \d_strd, \d_strd, #1 lsl \s_strd, \s_strd, #1 4: vld1.8 {d4}, [\src], \s_strd vld1.8 {d6}, [\sr2], \s_strd vext.8 d5, d4, d4, #1 vext.8 d7, d6, d6, #1 vtrn.32 q2, q3 subs \h, \h, #2 vmull.u8 q3, d4, d0 vmlal.u8 q3, d5, d1 .ifc \type, put vqrshrn.u16 d4, q3, #4 vst1.32 {d4[0]}, [\dst, :32], \d_strd vst1.32 {d4[1]}, [\ds2, :32], \d_strd .else vst1.16 {d6}, [\dst, :64], \d_strd vst1.16 {d7}, [\ds2, :64], \d_strd .endif bgt 4b pop {r4-r11,pc} 80: // 8xN h add \ds2, \dst, \d_strd add \sr2, \src, \s_strd lsl \d_strd, \d_strd, #1 lsl \s_strd, \s_strd, #1 8: vld1.8 {q8}, [\src], \s_strd vld1.8 {q10}, [\sr2], \s_strd vext.8 q9, q8, q8, #1 vext.8 q11, q10, q10, #1 subs \h, \h, #2 vmull.u8 q8, d16, d0 vmull.u8 q10, d20, d0 vmlal.u8 q8, d18, d1 vmlal.u8 q10, d22, d1 .ifc \type, put vqrshrn.u16 d16, q8, #4 vqrshrn.u16 d18, q10, #4 vst1.8 {d16}, [\dst, :64], \d_strd vst1.8 {d18}, [\ds2, :64], \d_strd .else vst1.16 {q8}, [\dst, :128], \d_strd vst1.16 {q10}, [\ds2, :128], \d_strd .endif bgt 8b pop {r4-r11,pc} 160: 320: 640: 1280: // 16xN, 32xN, ... h add \ds2, \dst, \d_strd add \sr2, \src, \s_strd lsl \s_strd, \s_strd, #1 sub \s_strd, \s_strd, \w sub \s_strd, \s_strd, #8 .ifc \type, put lsl \d_strd, \d_strd, #1 sub \d_strd, \d_strd, \w .endif 161: vld1.8 {d16}, [\src]! vld1.8 {d22}, [\sr2]! mov \mx, \w 16: vld1.8 {d17,d18}, [\src]! vld1.8 {d23,d24}, [\sr2]! vext.8 q10, q8, q9, #1 vext.8 q13, q11, q12, #1 vmull.u8 q2, d16, d0 vmull.u8 q3, d17, d0 vmull.u8 q14, d22, d0 vmull.u8 q15, d23, d0 vmlal.u8 q2, d20, d1 vmlal.u8 q3, d21, d1 vmlal.u8 q14, d26, d1 vmlal.u8 q15, d27, d1 subs \mx, \mx, #16 .ifc \type, put vqrshrn.u16 d4, q2, #4 vqrshrn.u16 d5, q3, #4 vqrshrn.u16 d28, q14, #4 vqrshrn.u16 d29, q15, #4 vst1.8 {q2}, [\dst, :128]! vst1.8 {q14}, [\ds2, :128]! .else vst1.16 {q2, q3}, [\dst, :128]! vst1.16 {q14, q15}, [\ds2, :128]! .endif ble 9f vmov d16, d18 vmov d22, d24 b 16b 9: add \dst, \dst, \d_strd add \ds2, \ds2, \d_strd add \src, \src, \s_strd add \sr2, \sr2, \s_strd subs \h, \h, #2 bgt 161b pop {r4-r11,pc} L(\type\()_bilin_v): cmp \h, #4 adr r9, L(\type\()_bilin_v_tbl) ldr r8, [r9, r8, lsl #2] add r9, r9, r8 bx r9 .align 2 L(\type\()_bilin_v_tbl): .word 1280f - L(\type\()_bilin_v_tbl) + CONFIG_THUMB .word 640f - L(\type\()_bilin_v_tbl) + CONFIG_THUMB .word 320f - L(\type\()_bilin_v_tbl) + CONFIG_THUMB .word 160f - L(\type\()_bilin_v_tbl) + CONFIG_THUMB .word 80f - L(\type\()_bilin_v_tbl) + CONFIG_THUMB .word 40f - L(\type\()_bilin_v_tbl) + CONFIG_THUMB .word 20f - L(\type\()_bilin_v_tbl) + CONFIG_THUMB 20: // 2xN v .ifc \type, put cmp \h, #2 add \ds2, \dst, \d_strd add \sr2, \src, \s_strd lsl \s_strd, \s_strd, #1 lsl \d_strd, \d_strd, #1 // 2x2 v vld1.16 {d16[]}, [\src], \s_strd bgt 24f 22: vld1.16 {d17[]}, [\sr2], \s_strd vld1.16 {d18[]}, [\src], \s_strd vext.8 d16, d16, d17, #6 vext.8 d17, d17, d18, #6 vmull.u8 q2, d16, d2 vmlal.u8 q2, d17, d3 vqrshrn.u16 d4, q2, #4 vst1.16 {d4[0]}, [\dst, :16] vst1.16 {d4[1]}, [\ds2, :16] pop {r4-r11,pc} 24: // 2x4, 2x6, 2x8, ... v vld1.16 {d17[]}, [\sr2], \s_strd vld1.16 {d18[]}, [\src], \s_strd vld1.16 {d19[]}, [\sr2], \s_strd vld1.16 {d20[]}, [\src], \s_strd sub \h, \h, #4 vext.8 d16, d16, d17, #6 vext.8 d17, d17, d18, #6 vext.8 d18, d18, d19, #6 vext.8 d19, d19, d20, #6 vtrn.32 d16, d18 vtrn.32 d17, d19 vmull.u8 q2, d16, d2 vmlal.u8 q2, d17, d3 cmp \h, #2 vqrshrn.u16 d4, q2, #4 vst1.16 {d4[0]}, [\dst, :16], \d_strd vst1.16 {d4[1]}, [\ds2, :16], \d_strd vst1.16 {d4[2]}, [\dst, :16], \d_strd vst1.16 {d4[3]}, [\ds2, :16], \d_strd blt 0f vmov d16, d20 beq 22b b 24b 0: pop {r4-r11,pc} .endif 40: // 4xN v add \ds2, \dst, \d_strd add \sr2, \src, \s_strd lsl \s_strd, \s_strd, #1 lsl \d_strd, \d_strd, #1 vld1.32 {d16[]}, [\src], \s_strd 4: vld1.32 {d17[]}, [\sr2], \s_strd vld1.32 {d18[]}, [\src], \s_strd vext.8 d16, d16, d17, #4 vext.8 d17, d17, d18, #4 vmull.u8 q2, d16, d2 vmlal.u8 q2, d17, d3 subs \h, \h, #2 .ifc \type, put vqrshrn.u16 d4, q2, #4 vst1.32 {d4[0]}, [\dst, :32], \d_strd vst1.32 {d4[1]}, [\ds2, :32], \d_strd .else vst1.16 {d4}, [\dst, :64], \d_strd vst1.16 {d5}, [\ds2, :64], \d_strd .endif ble 0f vmov d16, d18 b 4b 0: pop {r4-r11,pc} 80: // 8xN v add \ds2, \dst, \d_strd add \sr2, \src, \s_strd lsl \s_strd, \s_strd, #1 lsl \d_strd, \d_strd, #1 vld1.8 {d16}, [\src], \s_strd 8: vld1.8 {d17}, [\sr2], \s_strd vld1.8 {d18}, [\src], \s_strd vmull.u8 q2, d16, d2 vmull.u8 q3, d17, d2 vmlal.u8 q2, d17, d3 vmlal.u8 q3, d18, d3 subs \h, \h, #2 .ifc \type, put vqrshrn.u16 d4, q2, #4 vqrshrn.u16 d6, q3, #4 vst1.8 {d4}, [\dst, :64], \d_strd vst1.8 {d6}, [\ds2, :64], \d_strd .else vst1.16 {q2}, [\dst, :128], \d_strd vst1.16 {q3}, [\ds2, :128], \d_strd .endif ble 0f vmov d16, d18 b 8b 0: pop {r4-r11,pc} 160: // 16xN, 32xN, ... 320: 640: 1280: mov \my, \h 1: add \ds2, \dst, \d_strd add \sr2, \src, \s_strd lsl \s_strd, \s_strd, #1 lsl \d_strd, \d_strd, #1 vld1.8 {q8}, [\src], \s_strd 2: vld1.8 {q9}, [\sr2], \s_strd vld1.8 {q10}, [\src], \s_strd vmull.u8 q12, d16, d2 vmull.u8 q13, d17, d2 vmull.u8 q14, d18, d2 vmull.u8 q15, d19, d2 vmlal.u8 q12, d18, d3 vmlal.u8 q13, d19, d3 vmlal.u8 q14, d20, d3 vmlal.u8 q15, d21, d3 subs \h, \h, #2 .ifc \type, put vqrshrn.u16 d24, q12, #4 vqrshrn.u16 d25, q13, #4 vqrshrn.u16 d28, q14, #4 vqrshrn.u16 d29, q15, #4 vst1.8 {q12}, [\dst, :128], \d_strd vst1.8 {q14}, [\ds2, :128], \d_strd .else vst1.16 {q12, q13}, [\dst, :128], \d_strd vst1.16 {q14, q15}, [\ds2, :128], \d_strd .endif ble 9f vmov q8, q10 b 2b 9: subs \w, \w, #16 ble 0f asr \s_strd, \s_strd, #1 asr \d_strd, \d_strd, #1 mls \src, \s_strd, \my, \src mls \dst, \d_strd, \my, \dst sub \src, \src, \s_strd, lsl #1 mov \h, \my add \src, \src, #16 .ifc \type, put add \dst, \dst, #16 .else add \dst, \dst, #32 .endif b 1b 0: pop {r4-r11,pc} L(\type\()_bilin_hv): vmovl.u8 q2, d2 vmovl.u8 q3, d3 adr r9, L(\type\()_bilin_hv_tbl) ldr r8, [r9, r8, lsl #2] add r9, r9, r8 bx r9 .align 2 L(\type\()_bilin_hv_tbl): .word 1280f - L(\type\()_bilin_hv_tbl) + CONFIG_THUMB .word 640f - L(\type\()_bilin_hv_tbl) + CONFIG_THUMB .word 320f - L(\type\()_bilin_hv_tbl) + CONFIG_THUMB .word 160f - L(\type\()_bilin_hv_tbl) + CONFIG_THUMB .word 80f - L(\type\()_bilin_hv_tbl) + CONFIG_THUMB .word 40f - L(\type\()_bilin_hv_tbl) + CONFIG_THUMB .word 20f - L(\type\()_bilin_hv_tbl) + CONFIG_THUMB 20: // 2xN hv .ifc \type, put add \sr2, \src, \s_strd add \ds2, \dst, \d_strd lsl \s_strd, \s_strd, #1 lsl \d_strd, \d_strd, #1 vld1.32 {d28[]}, [\src], \s_strd vext.8 d29, d28, d28, #1 vmull.u8 q8, d28, d0 vmlal.u8 q8, d29, d1 2: vld1.32 {d28[]}, [\sr2], \s_strd vld1.32 {d30[]}, [\src], \s_strd vext.8 d29, d28, d28, #1 vext.8 d31, d30, d30, #1 vtrn.16 d28, d30 vtrn.16 d29, d31 vmull.u8 q9, d28, d0 vmlal.u8 q9, d29, d1 vtrn.32 d16, d18 vmul.u16 d20, d16, d4 vmla.u16 d20, d19, d6 vqrshrn.u16 d20, q10, #8 subs \h, \h, #2 vst1.16 {d20[0]}, [\dst, :16], \d_strd vst1.16 {d20[1]}, [\ds2, :16], \d_strd ble 0f vtrn.32 d19, d16 b 2b 0: pop {r4-r11,pc} .endif 40: // 4xN hv add \sr2, \src, \s_strd add \ds2, \dst, \d_strd lsl \s_strd, \s_strd, #1 lsl \d_strd, \d_strd, #1 vld1.8 {d28}, [\src], \s_strd vext.8 d29, d28, d28, #1 vmull.u8 q8, d28, d0 vmlal.u8 q8, d29, d1 4: vld1.8 {d28}, [\sr2], \s_strd vld1.8 {d30}, [\src], \s_strd vext.8 d29, d28, d28, #1 vext.8 d31, d30, d30, #1 vtrn.32 d28, d30 vtrn.32 d29, d31 vmull.u8 q9, d28, d0 vmlal.u8 q9, d29, d1 vmov d17, d18 vmul.u16 q10, q8, q2 vmla.u16 q10, q9, q3 subs \h, \h, #2 .ifc \type, put vqrshrn.u16 d20, q10, #8 vst1.32 {d20[0]}, [\dst, :32], \d_strd vst1.32 {d20[1]}, [\ds2, :32], \d_strd .else vrshr.u16 q10, q10, #4 vst1.16 {d20}, [\dst, :64], \d_strd vst1.16 {d21}, [\ds2, :64], \d_strd .endif ble 0f vmov d16, d19 b 4b 0: pop {r4-r11,pc} 80: // 8xN, 16xN, ... hv 160: 320: 640: 1280: mov \my, \h 1: add \sr2, \src, \s_strd add \ds2, \dst, \d_strd lsl \s_strd, \s_strd, #1 lsl \d_strd, \d_strd, #1 vld1.8 {q12}, [\src], \s_strd vext.8 q13, q12, q12, #1 vmull.u8 q8, d24, d0 vmlal.u8 q8, d26, d1 2: vld1.8 {q12}, [\sr2], \s_strd vld1.8 {q14}, [\src], \s_strd vext.8 q13, q12, q12, #1 vext.8 q15, q14, q14, #1 vmull.u8 q9, d24, d0 vmlal.u8 q9, d26, d1 vmull.u8 q10, d28, d0 vmlal.u8 q10, d30, d1 vmul.u16 q8, q8, q2 vmla.u16 q8, q9, q3 vmul.u16 q9, q9, q2 vmla.u16 q9, q10, q3 subs \h, \h, #2 .ifc \type, put vqrshrn.u16 d16, q8, #8 vqrshrn.u16 d18, q9, #8 vst1.8 {d16}, [\dst, :64], \d_strd vst1.8 {d18}, [\ds2, :64], \d_strd .else vrshr.u16 q8, q8, #4 vrshr.u16 q9, q9, #4 vst1.16 {q8}, [\dst, :128], \d_strd vst1.16 {q9}, [\ds2, :128], \d_strd .endif ble 9f vmov q8, q10 b 2b 9: subs \w, \w, #8 ble 0f asr \s_strd, \s_strd, #1 asr \d_strd, \d_strd, #1 mls \src, \s_strd, \my, \src mls \dst, \d_strd, \my, \dst sub \src, \src, \s_strd, lsl #1 mov \h, \my add \src, \src, #8 .ifc \type, put add \dst, \dst, #8 .else add \dst, \dst, #16 .endif b 1b 0: pop {r4-r11,pc} endfunc .endm filter_fn put, r0, r1, r2, r3, r4, r5, r6, r7, r8, r9, 10 filter_fn prep, r0, r7, r1, r2, r3, r4, r5, r6, r8, r9, 6 .macro load_filter_ptr src asr r12, \src, #10 add r12, r11, r12, lsl #3 .endm .macro load_filter_coef dst, src, inc add \src, \src, \inc vld1.8 {\dst}, [r12, :64] .endm .macro load_filter_row dst, src, inc load_filter_ptr \src load_filter_coef \dst, \src, \inc .endm function warp_filter_horz_neon load_filter_ptr r5 // filter 0 vld1.16 {q7}, [r2], r3 vmov.i8 q6, #128 load_filter_coef d0, r5, r7 // filter 0 load_filter_row d1, r5, r7 // filter 1 load_filter_row d2, r5, r7 // filter 2 load_filter_ptr r5 // filter 3 veor q7, q7, q6 // subtract by 128 to allow using vmull load_filter_coef d3, r5, r7 // filter 3 vext.8 d12, d14, d15, #1 // filter 1 pixels vext.8 d13, d14, d15, #2 // filter 2 pixels load_filter_ptr r5 // filter 4 vmull.s8 q2, d14, d0 // filter 0 output vmull.s8 q3, d12, d1 // filter 1 output load_filter_coef d0, r5, r7 // filter 4 load_filter_ptr r5 // filter 5 vext.8 d12, d14, d15, #3 // filter 3 pixels vmull.s8 q4, d13, d2 // filter 2 output vext.8 d13, d14, d15, #4 // filter 4 pixels vpadd.i16 d4, d4, d5 // pixel 0 (4x16) vpadd.i16 d5, d6, d7 // pixel 1 (4x16) load_filter_coef d1, r5, r7 // filter 5 load_filter_ptr r5 // filter 6 vmull.s8 q5, d12, d3 // filter 3 output vext.8 d12, d14, d15, #5 // filter 5 pixels vmull.s8 q3, d13, d0 // filter 4 output load_filter_coef d0, r5, r7 // filter 6 vext.8 d13, d14, d15, #6 // filter 6 pixels load_filter_ptr r5 // filter 7 vpadd.i16 d8, d8, d9 // pixel 2 (4x16) vpadd.i16 d9, d10, d11 // pixel 3 (4x16) vmull.s8 q5, d12, d1 // filter 5 output load_filter_coef d1, r5, r7 // filter 7 vext.8 d14, d14, d15, #7 // filter 7 pixels vpadd.i16 d6, d6, d7 // pixel 4 (4x16) vpadd.i16 d10, d10, d11 // pixel 5 (4x16) vmull.s8 q6, d13, d0 // filter 6 output vmull.s8 q7, d14, d1 // filter 7 output sub r5, r5, r7, lsl #3 vpadd.i16 d4, d4, d5 // pixel 0,1 (2x16) vpadd.i16 d5, d8, d9 // pixel 2,3 (2x16) vpadd.i16 d12, d12, d13 // pixel 6 (4x16) vpadd.i16 d14, d14, d15 // pixel 7 (4x16) vpadd.i16 d6, d6, d10 // pixel 4,5 (2x16) vpadd.i16 d10, d12, d14 // pixel 6,7 (2x16) vpadd.i16 d4, d4, d5 // pixel 0-3 vpadd.i16 d5, d6, d10 // pixel 4-7 add r5, r5, r8 bx lr endfunc // void dav1d_warp_affine_8x8_8bpc_neon( // pixel *dst, const ptrdiff_t dst_stride, // const pixel *src, const ptrdiff_t src_stride, // const int16_t *const abcd, int mx, int my) .macro warp t, shift function warp_affine_8x8\t\()_8bpc_neon, export=1 push {r4-r11,lr} vpush {q4-q7} ldrd r4, r5, [sp, #100] ldr r6, [sp, #108] ldrd r8, r9, [r4] sxth r7, r8 asr r8, r8, #16 asr r4, r9, #16 sxth r9, r9 mov r10, #8 sub r2, r2, r3, lsl #1 sub r2, r2, r3 sub r2, r2, #3 movrel r11, X(mc_warp_filter), 64*8 .ifnb \t lsl r1, r1, #1 .endif add r5, r5, #512 add r6, r6, #512 bl warp_filter_horz_neon vrshr.s16 q8, q2, #3 bl warp_filter_horz_neon vrshr.s16 q9, q2, #3 bl warp_filter_horz_neon vrshr.s16 q10, q2, #3 bl warp_filter_horz_neon vrshr.s16 q11, q2, #3 bl warp_filter_horz_neon vrshr.s16 q12, q2, #3 bl warp_filter_horz_neon vrshr.s16 q13, q2, #3 bl warp_filter_horz_neon vrshr.s16 q14, q2, #3 1: bl warp_filter_horz_neon vrshr.s16 q15, q2, #3 load_filter_row d8, r6, r9 load_filter_row d9, r6, r9 load_filter_row d10, r6, r9 load_filter_row d11, r6, r9 load_filter_row d12, r6, r9 load_filter_row d13, r6, r9 load_filter_row d14, r6, r9 load_filter_row d15, r6, r9 transpose_8x8b q4, q5, q6, q7, d8, d9, d10, d11, d12, d13, d14, d15 vmovl.s8 q1, d8 vmovl.s8 q2, d9 vmovl.s8 q3, d10 vmovl.s8 q4, d11 vmovl.s8 q5, d12 vmovl.s8 q6, d13 sub r6, r6, r9, lsl #3 // This ordering of vmull/vmlal is highly beneficial for // Cortex A8/A9/A53 here, but harmful for Cortex A7. vmull.s16 q0, d16, d2 vmlal.s16 q0, d18, d4 vmlal.s16 q0, d20, d6 vmlal.s16 q0, d22, d8 vmlal.s16 q0, d24, d10 vmlal.s16 q0, d26, d12 vmull.s16 q1, d17, d3 vmlal.s16 q1, d19, d5 vmlal.s16 q1, d21, d7 vmlal.s16 q1, d23, d9 vmlal.s16 q1, d25, d11 vmlal.s16 q1, d27, d13 vmovl.s8 q2, d14 vmovl.s8 q3, d15 vmlal.s16 q0, d28, d4 vmlal.s16 q0, d30, d6 vmlal.s16 q1, d29, d5 vmlal.s16 q1, d31, d7 .ifb \t vmov.i16 q7, #128 .else vmov.i16 q7, #0x800 .endif vmov q8, q9 vmov q9, q10 vqrshrn.s32 d0, q0, #\shift vmov q10, q11 vqrshrn.s32 d1, q1, #\shift vmov q11, q12 vadd.i16 q0, q0, q7 vmov q12, q13 .ifb \t vqmovun.s16 d0, q0 .endif vmov q13, q14 vmov q14, q15 subs r10, r10, #1 .ifnb \t vst1.16 {q0}, [r0, :128], r1 .else vst1.8 {d0}, [r0, :64], r1 .endif add r6, r6, r4 bgt 1b vpop {q4-q7} pop {r4-r11,pc} endfunc .endm warp , 11 warp t, 7 // void dav1d_emu_edge_8bpc_neon( // const intptr_t bw, const intptr_t bh, // const intptr_t iw, const intptr_t ih, // const intptr_t x, const intptr_t y, // pixel *dst, const ptrdiff_t dst_stride, // const pixel *ref, const ptrdiff_t ref_stride) function emu_edge_8bpc_neon, export=1 push {r4-r11,lr} ldrd r4, r5, [sp, #36] ldrd r6, r7, [sp, #44] ldrd r8, r9, [sp, #52] // ref += iclip(y, 0, ih - 1) * PXSTRIDE(ref_stride) // ref += iclip(x, 0, iw - 1) sub r12, r3, #1 // ih - 1 cmp r5, r3 sub lr, r2, #1 // iw - 1 it lt movlt r12, r5 // min(y, ih - 1) cmp r4, r2 bic r12, r12, r12, asr #31 // max(min(y, ih - 1), 0) it lt movlt lr, r4 // min(x, iw - 1) bic lr, lr, lr, asr #31 // max(min(x, iw - 1), 0) mla r8, r12, r9, r8 // ref += iclip() * stride add r8, r8, lr // ref += iclip() // bottom_ext = iclip(y + bh - ih, 0, bh - 1) // top_ext = iclip(-y, 0, bh - 1) add r10, r5, r1 // y + bh neg r5, r5 // -y sub r10, r10, r3 // y + bh - ih sub r12, r1, #1 // bh - 1 cmp r10, r1 bic r5, r5, r5, asr #31 // max(-y, 0) it ge movge r10, r12 // min(y + bh - ih, bh-1) cmp r5, r1 bic r10, r10, r10, asr #31 // max(min(y + bh - ih, bh-1), 0) it ge movge r5, r12 // min(max(-y, 0), bh-1) // right_ext = iclip(x + bw - iw, 0, bw - 1) // left_ext = iclip(-x, 0, bw - 1) add r11, r4, r0 // x + bw neg r4, r4 // -x sub r11, r11, r2 // x + bw - iw sub lr, r0, #1 // bw - 1 cmp r11, r0 bic r4, r4, r4, asr #31 // max(-x, 0) it ge movge r11, lr // min(x + bw - iw, bw-1) cmp r4, r0 bic r11, r11, r11, asr #31 // max(min(x + bw - iw, bw-1), 0) it ge movge r4, lr // min(max(-x, 0), bw - 1) // center_h = bh - top_ext - bottom_ext // dst += top_ext * PXSTRIDE(dst_stride) // center_w = bw - left_ext - right_ext sub r1, r1, r5 // bh - top_ext mla r6, r5, r7, r6 sub r2, r0, r4 // bw - left_ext sub r1, r1, r10 // center_h = bh - top_ext - bottom_ext sub r2, r2, r11 // center_w = bw - left_ext - right_ext mov r0, r6 // backup of dst .macro v_loop need_left, need_right 0: .if \need_left vld1.8 {d0[], d1[]}, [r8] mov r12, r6 // out = dst mov r3, r4 1: subs r3, r3, #16 vst1.8 {q0}, [r12, :128]! bgt 1b .endif mov lr, r8 add r12, r6, r4 // out = dst + left_ext mov r3, r2 1: vld1.8 {q0, q1}, [lr]! subs r3, r3, #32 .if \need_left vst1.8 {q0, q1}, [r12]! .else vst1.8 {q0, q1}, [r12, :128]! .endif bgt 1b .if \need_right add r3, r8, r2 // in + center_w sub r3, r3, #1 // in + center_w - 1 add r12, r6, r4 // dst + left_ext vld1.8 {d0[], d1[]}, [r3] add r12, r12, r2 // out = dst + left_ext + center_w mov r3, r11 1: subs r3, r3, #16 vst1.8 {q0}, [r12]! bgt 1b .endif subs r1, r1, #1 // center_h-- add r6, r6, r7 add r8, r8, r9 bgt 0b .endm cmp r4, #0 beq 2f // need_left cmp r11, #0 beq 3f // need_left + need_right v_loop 1, 1 b 5f 2: // !need_left cmp r11, #0 beq 4f // !need_left + need_right v_loop 0, 1 b 5f 3: // need_left + !need_right v_loop 1, 0 b 5f 4: // !need_left + !need_right v_loop 0, 0 5: cmp r10, #0 // Storing the original dst in r0 overwrote bw, recalculate it here add r2, r2, r4 // center_w + left_ext add r2, r2, r11 // bw = center_w + left_ext + right_ext beq 3f // need_bottom sub r8, r6, r7 // ref = dst - stride mov r4, r2 1: vld1.8 {q0, q1}, [r8, :128]! mov r3, r10 2: subs r3, r3, #1 vst1.8 {q0, q1}, [r6, :128], r7 bgt 2b mls r6, r7, r10, r6 // dst -= bottom_ext * stride subs r4, r4, #32 // bw -= 32 add r6, r6, #32 // dst += 32 bgt 1b 3: cmp r5, #0 beq 3f // need_top mls r6, r7, r5, r0 // dst = stored_dst - top_ext * stride 1: vld1.8 {q0, q1}, [r0, :128]! mov r3, r5 2: subs r3, r3, #1 vst1.8 {q0, q1}, [r6, :128], r7 bgt 2b mls r6, r7, r5, r6 // dst -= top_ext * stride subs r2, r2, #32 // bw -= 32 add r6, r6, #32 // dst += 32 bgt 1b 3: pop {r4-r11,pc} endfunc