/* * Copyright © 2018, VideoLAN and dav1d authors * Copyright © 2020, Martin Storsjo * Copyright © 2019, B Krishnan Iyer * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, this * list of conditions and the following disclaimer. * * 2. Redistributions in binary form must reproduce the above copyright notice, * this list of conditions and the following disclaimer in the documentation * and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include "src/arm/asm.S" #include "util.S" // void ipred_dc_128_8bpc_neon(pixel *dst, const ptrdiff_t stride, // const pixel *const topleft, // const int width, const int height, const int a, // const int max_width, const int max_height); function ipred_dc_128_8bpc_neon, export=1 push {r4, lr} ldr r4, [sp, #8] clz r3, r3 adr r2, L(ipred_dc_128_tbl) sub r3, r3, #25 ldr r3, [r2, r3, lsl #2] vmov.i8 q0, #128 add r2, r2, r3 add r12, r0, r1 lsl r1, r1, #1 bx r2 .align 2 L(ipred_dc_128_tbl): .word 640f - L(ipred_dc_128_tbl) + CONFIG_THUMB .word 320f - L(ipred_dc_128_tbl) + CONFIG_THUMB .word 16f - L(ipred_dc_128_tbl) + CONFIG_THUMB .word 8f - L(ipred_dc_128_tbl) + CONFIG_THUMB .word 4f - L(ipred_dc_128_tbl) + CONFIG_THUMB 4: vst1.32 {d0[0]}, [r0, :32], r1 vst1.32 {d0[0]}, [r12, :32], r1 subs r4, r4, #4 vst1.32 {d0[0]}, [r0, :32], r1 vst1.32 {d0[0]}, [r12, :32], r1 bgt 4b pop {r4, pc} 8: vst1.8 {d0}, [r0, :64], r1 vst1.8 {d0}, [r12, :64], r1 subs r4, r4, #4 vst1.8 {d0}, [r0, :64], r1 vst1.8 {d0}, [r12, :64], r1 bgt 8b pop {r4, pc} 16: vst1.8 {d0, d1}, [r0, :128], r1 vst1.8 {d0, d1}, [r12, :128], r1 subs r4, r4, #4 vst1.8 {d0, d1}, [r0, :128], r1 vst1.8 {d0, d1}, [r12, :128], r1 bgt 16b pop {r4, pc} 320: vmov.i8 q1, #128 32: vst1.8 {d0, d1, d2, d3}, [r0, :128], r1 vst1.8 {d0, d1, d2, d3}, [r12, :128], r1 subs r4, r4, #4 vst1.8 {d0, d1, d2, d3}, [r0, :128], r1 vst1.8 {d0, d1, d2, d3}, [r12, :128], r1 bgt 32b pop {r4, pc} 640: vmov.i8 q1, #128 sub r1, r1, #32 64: vst1.8 {d0, d1, d2, d3}, [r0, :128]! vst1.8 {d0, d1, d2, d3}, [r12, :128]! vst1.8 {d0, d1, d2, d3}, [r0, :128], r1 vst1.8 {d0, d1, d2, d3}, [r12, :128], r1 subs r4, r4, #4 vst1.8 {d0, d1, d2, d3}, [r0, :128]! vst1.8 {d0, d1, d2, d3}, [r12, :128]! vst1.8 {d0, d1, d2, d3}, [r0, :128], r1 vst1.8 {d0, d1, d2, d3}, [r12, :128], r1 bgt 64b pop {r4, pc} endfunc // void ipred_v_8bpc_neon(pixel *dst, const ptrdiff_t stride, // const pixel *const topleft, // const int width, const int height, const int a, // const int max_width, const int max_height); function ipred_v_8bpc_neon, export=1 push {r4, lr} ldr lr, [sp, #8] clz r3, r3 adr r4, L(ipred_v_tbl) sub r3, r3, #25 ldr r3, [r4, r3, lsl #2] add r2, r2, #1 add r4, r4, r3 add r12, r0, r1 lsl r1, r1, #1 bx r4 .align 2 L(ipred_v_tbl): .word 640f - L(ipred_v_tbl) + CONFIG_THUMB .word 320f - L(ipred_v_tbl) + CONFIG_THUMB .word 160f - L(ipred_v_tbl) + CONFIG_THUMB .word 80f - L(ipred_v_tbl) + CONFIG_THUMB .word 40f - L(ipred_v_tbl) + CONFIG_THUMB 40: vld1.32 {d0[]}, [r2] 4: vst1.32 {d0[0]}, [r0, :32], r1 vst1.32 {d0[0]}, [r12, :32], r1 subs lr, lr, #4 vst1.32 {d0[0]}, [r0, :32], r1 vst1.32 {d0[0]}, [r12, :32], r1 bgt 4b pop {r4, pc} 80: vld1.8 {d0}, [r2] 8: vst1.8 {d0}, [r0, :64], r1 vst1.8 {d0}, [r12, :64], r1 subs lr, lr, #4 vst1.8 {d0}, [r0, :64], r1 vst1.8 {d0}, [r12, :64], r1 bgt 8b pop {r4, pc} 160: vld1.8 {q0}, [r2] 16: vst1.8 {d0, d1}, [r0, :128], r1 vst1.8 {d0, d1}, [r12, :128], r1 subs lr, lr, #4 vst1.8 {d0, d1}, [r0, :128], r1 vst1.8 {d0, d1}, [r12, :128], r1 bgt 16b pop {r4, pc} 320: vld1.8 {q0, q1}, [r2] 32: vst1.8 {d0, d1, d2, d3}, [r0, :128], r1 vst1.8 {d0, d1, d2, d3}, [r12, :128], r1 subs lr, lr, #4 vst1.8 {d0, d1, d2, d3}, [r0, :128], r1 vst1.8 {d0, d1, d2, d3}, [r12, :128], r1 bgt 32b pop {r4, pc} 640: vld1.8 {q0, q1}, [r2]! sub r1, r1, #32 vld1.8 {q2, q3}, [r2] 64: vst1.8 {d0, d1, d2, d3}, [r0, :128]! vst1.8 {d0, d1, d2, d3}, [r12, :128]! vst1.8 {d4, d5, d6, d7}, [r0, :128], r1 vst1.8 {d4, d5, d6, d7}, [r12, :128], r1 subs lr, lr, #4 vst1.8 {d0, d1, d2, d3}, [r0, :128]! vst1.8 {d0, d1, d2, d3}, [r12, :128]! vst1.8 {d4, d5, d6, d7}, [r0, :128], r1 vst1.8 {d4, d5, d6, d7}, [r12, :128], r1 bgt 64b pop {r4, pc} endfunc // void ipred_h_8bpc_neon(pixel *dst, const ptrdiff_t stride, // const pixel *const topleft, // const int width, const int height, const int a, // const int max_width, const int max_height); function ipred_h_8bpc_neon, export=1 push {r4-r5, lr} ldr r4, [sp, #12] clz r3, r3 adr r5, L(ipred_h_tbl) sub r3, r3, #25 ldr r3, [r5, r3, lsl #2] sub r2, r2, #4 mov lr, #-4 add r5, r5, r3 add r12, r0, r1 lsl r1, r1, #1 bx r5 .align 2 L(ipred_h_tbl): .word 640f - L(ipred_h_tbl) + CONFIG_THUMB .word 320f - L(ipred_h_tbl) + CONFIG_THUMB .word 160f - L(ipred_h_tbl) + CONFIG_THUMB .word 8f - L(ipred_h_tbl) + CONFIG_THUMB .word 4f - L(ipred_h_tbl) + CONFIG_THUMB 4: vld4.8 {d0[], d1[], d2[], d3[]}, [r2, :32], lr vst1.32 {d3[0]}, [r0, :32], r1 vst1.32 {d2[0]}, [r12, :32], r1 subs r4, r4, #4 vst1.32 {d1[0]}, [r0, :32], r1 vst1.32 {d0[0]}, [r12, :32], r1 bgt 4b pop {r4-r5, pc} 8: vld4.8 {d0[], d1[], d2[], d3[]}, [r2, :32], lr vst1.8 {d3}, [r0, :64], r1 vst1.8 {d2}, [r12, :64], r1 subs r4, r4, #4 vst1.8 {d1}, [r0, :64], r1 vst1.8 {d0}, [r12, :64], r1 bgt 8b pop {r4-r5, pc} 160: add r2, r2, #3 mov lr, #-1 16: vld1.8 {d0[], d1[]}, [r2], lr subs r4, r4, #4 vld1.8 {d2[], d3[]}, [r2], lr vst1.8 {q0}, [r0, :128], r1 vld1.8 {d4[], d5[]}, [r2], lr vst1.8 {q1}, [r12, :128], r1 vld1.8 {d6[], d7[]}, [r2], lr vst1.8 {q2}, [r0, :128], r1 vst1.8 {q3}, [r12, :128], r1 bgt 16b pop {r4-r5, pc} 320: add r2, r2, #3 mov lr, #-1 sub r1, r1, #16 32: vld1.8 {d0[], d1[]}, [r2], lr subs r4, r4, #4 vld1.8 {d2[], d3[]}, [r2], lr vst1.8 {q0}, [r0, :128]! vld1.8 {d4[], d5[]}, [r2], lr vst1.8 {q1}, [r12, :128]! vld1.8 {d6[], d7[]}, [r2], lr vst1.8 {q0}, [r0, :128], r1 vst1.8 {q1}, [r12, :128], r1 vst1.8 {q2}, [r0, :128]! vst1.8 {q3}, [r12, :128]! vst1.8 {q2}, [r0, :128], r1 vst1.8 {q3}, [r12, :128], r1 bgt 32b pop {r4-r5, pc} 640: add r2, r2, #3 mov lr, #-1 sub r1, r1, #48 64: vld1.8 {d0[], d1[]}, [r2], lr subs r4, r4, #4 vld1.8 {d2[], d3[]}, [r2], lr vst1.8 {q0}, [r0, :128]! vld1.8 {d4[], d5[]}, [r2], lr vst1.8 {q1}, [r12, :128]! vld1.8 {d6[], d7[]}, [r2], lr vst1.8 {q0}, [r0, :128]! vst1.8 {q1}, [r12, :128]! vst1.8 {q0}, [r0, :128]! vst1.8 {q1}, [r12, :128]! vst1.8 {q0}, [r0, :128], r1 vst1.8 {q1}, [r12, :128], r1 vst1.8 {q2}, [r0, :128]! vst1.8 {q3}, [r12, :128]! vst1.8 {q2}, [r0, :128]! vst1.8 {q3}, [r12, :128]! vst1.8 {q2}, [r0, :128]! vst1.8 {q3}, [r12, :128]! vst1.8 {q2}, [r0, :128], r1 vst1.8 {q3}, [r12, :128], r1 bgt 64b pop {r4-r5, pc} endfunc // void ipred_dc_top_8bpc_neon(pixel *dst, const ptrdiff_t stride, // const pixel *const topleft, // const int width, const int height, const int a, // const int max_width, const int max_height); function ipred_dc_top_8bpc_neon, export=1 push {r4-r5, lr} ldr r4, [sp, #12] clz r3, r3 adr r5, L(ipred_dc_top_tbl) sub r3, r3, #25 ldr r3, [r5, r3, lsl #2] add r2, r2, #1 add r5, r5, r3 add r12, r0, r1 lsl r1, r1, #1 bx r5 .align 2 L(ipred_dc_top_tbl): .word 640f - L(ipred_dc_top_tbl) + CONFIG_THUMB .word 320f - L(ipred_dc_top_tbl) + CONFIG_THUMB .word 160f - L(ipred_dc_top_tbl) + CONFIG_THUMB .word 80f - L(ipred_dc_top_tbl) + CONFIG_THUMB .word 40f - L(ipred_dc_top_tbl) + CONFIG_THUMB 40: vld1.32 {d0[]}, [r2] vpaddl.u8 d0, d0 vpadd.u16 d0, d0 vrshrn.u16 d0, q0, #2 vdup.8 d0, d0[0] 4: vst1.32 {d0[0]}, [r0, :32], r1 vst1.32 {d0[0]}, [r12, :32], r1 subs r4, r4, #4 vst1.32 {d0[0]}, [r0, :32], r1 vst1.32 {d0[0]}, [r12, :32], r1 bgt 4b pop {r4-r5, pc} 80: vld1.8 {d0}, [r2] vpaddl.u8 d0, d0 vpadd.u16 d0, d0 vpadd.u16 d0, d0 vrshrn.u16 d0, q0, #3 vdup.8 d0, d0[0] 8: vst1.8 {d0}, [r0, :64], r1 vst1.8 {d0}, [r12, :64], r1 subs r4, r4, #4 vst1.8 {d0}, [r0, :64], r1 vst1.8 {d0}, [r12, :64], r1 bgt 8b pop {r4-r5, pc} 160: vld1.8 {d0, d1}, [r2] vaddl.u8 q0, d0, d1 vadd.u16 d0, d0, d1 vpadd.u16 d0, d0 vpadd.u16 d0, d0 vrshrn.u16 d0, q0, #4 vdup.8 q0, d0[0] 16: vst1.8 {d0, d1}, [r0, :128], r1 vst1.8 {d0, d1}, [r12, :128], r1 subs r4, r4, #4 vst1.8 {d0, d1}, [r0, :128], r1 vst1.8 {d0, d1}, [r12, :128], r1 bgt 16b pop {r4-r5, pc} 320: vld1.8 {d0, d1, d2, d3}, [r2] vaddl.u8 q0, d0, d1 vaddl.u8 q1, d2, d3 vadd.u16 q0, q0, q1 vadd.u16 d0, d0, d1 vpadd.u16 d0, d0 vpadd.u16 d0, d0 vrshrn.u16 d4, q0, #5 vdup.8 q0, d4[0] vdup.8 q1, d4[0] 32: vst1.8 {d0, d1, d2, d3}, [r0, :128], r1 vst1.8 {d0, d1, d2, d3}, [r12, :128], r1 subs r4, r4, #4 vst1.8 {d0, d1, d2, d3}, [r0, :128], r1 vst1.8 {d0, d1, d2, d3}, [r12, :128], r1 bgt 32b pop {r4-r5, pc} 640: vld1.8 {d0, d1, d2, d3}, [r2]! vaddl.u8 q0, d0, d1 vld1.8 {d4, d5, d6, d7}, [r2] vaddl.u8 q1, d2, d3 vaddl.u8 q2, d4, d5 vaddl.u8 q3, d6, d7 vadd.u16 q0, q0, q1 vadd.u16 q1, q2, q3 vadd.u16 q0, q0, q1 vadd.u16 d0, d0, d1 vpadd.u16 d0, d0 vpadd.u16 d0, d0 vrshrn.u16 d18, q0, #6 vdup.8 q0, d18[0] vdup.8 q1, d18[0] sub r1, r1, #32 64: vst1.8 {d0, d1, d2, d3}, [r0, :128]! vst1.8 {d0, d1, d2, d3}, [r12, :128]! vst1.8 {d0, d1, d2, d3}, [r0, :128], r1 vst1.8 {d0, d1, d2, d3}, [r12, :128], r1 subs r4, r4, #4 vst1.8 {d0, d1, d2, d3}, [r0, :128]! vst1.8 {d0, d1, d2, d3}, [r12, :128]! vst1.8 {d0, d1, d2, d3}, [r0, :128], r1 vst1.8 {d0, d1, d2, d3}, [r12, :128], r1 bgt 64b pop {r4-r5, pc} endfunc // void ipred_dc_left_8bpc_neon(pixel *dst, const ptrdiff_t stride, // const pixel *const topleft, // const int width, const int height, const int a, // const int max_width, const int max_height); function ipred_dc_left_8bpc_neon, export=1 push {r4-r5, lr} ldr r4, [sp, #12] sub r2, r2, r4 clz r3, r3 clz lr, r4 sub lr, lr, #25 adr r5, L(ipred_dc_left_tbl) sub r3, r3, #20 ldr r3, [r5, r3, lsl #2] ldr lr, [r5, lr, lsl #2] add r3, r5, r3 add r5, r5, lr add r12, r0, r1 lsl r1, r1, #1 bx r5 .align 2 L(ipred_dc_left_tbl): .word L(ipred_dc_left_h64) - L(ipred_dc_left_tbl) + CONFIG_THUMB .word L(ipred_dc_left_h32) - L(ipred_dc_left_tbl) + CONFIG_THUMB .word L(ipred_dc_left_h16) - L(ipred_dc_left_tbl) + CONFIG_THUMB .word L(ipred_dc_left_h8) - L(ipred_dc_left_tbl) + CONFIG_THUMB .word L(ipred_dc_left_h4) - L(ipred_dc_left_tbl) + CONFIG_THUMB .word L(ipred_dc_left_w64) - L(ipred_dc_left_tbl) + CONFIG_THUMB .word L(ipred_dc_left_w32) - L(ipred_dc_left_tbl) + CONFIG_THUMB .word L(ipred_dc_left_w16) - L(ipred_dc_left_tbl) + CONFIG_THUMB .word L(ipred_dc_left_w8) - L(ipred_dc_left_tbl) + CONFIG_THUMB .word L(ipred_dc_left_w4) - L(ipred_dc_left_tbl) + CONFIG_THUMB L(ipred_dc_left_h4): vld1.32 {d0[]}, [r2, :32] vpaddl.u8 d0, d0 vpadd.u16 d0, d0 vrshrn.u16 d0, q0, #2 vdup.8 q0, d0[0] bx r3 L(ipred_dc_left_w4): vst1.32 {d0[0]}, [r0, :32], r1 vst1.32 {d0[0]}, [r12, :32], r1 subs r4, r4, #4 vst1.32 {d0[0]}, [r0, :32], r1 vst1.32 {d0[0]}, [r12, :32], r1 bgt L(ipred_dc_left_w4) pop {r4-r5, pc} L(ipred_dc_left_h8): vld1.8 {d0}, [r2, :64] vpaddl.u8 d0, d0 vpadd.u16 d0, d0 vpadd.u16 d0, d0 vrshrn.u16 d0, q0, #3 vdup.8 q0, d0[0] bx r3 L(ipred_dc_left_w8): vst1.8 {d0}, [r0, :64], r1 vst1.8 {d0}, [r12, :64], r1 subs r4, r4, #4 vst1.8 {d0}, [r0, :64], r1 vst1.8 {d0}, [r12, :64], r1 bgt L(ipred_dc_left_w8) pop {r4-r5, pc} L(ipred_dc_left_h16): vld1.8 {d0, d1}, [r2, :128] vaddl.u8 q0, d0, d1 vadd.u16 d0, d0, d1 vpadd.u16 d0, d0 vpadd.u16 d0, d0 vrshrn.u16 d0, q0, #4 vdup.8 q0, d0[0] bx r3 L(ipred_dc_left_w16): vst1.8 {d0, d1}, [r0, :128], r1 vst1.8 {d0, d1}, [r12, :128], r1 subs r4, r4, #4 vst1.8 {d0, d1}, [r0, :128], r1 vst1.8 {d0, d1}, [r12, :128], r1 bgt L(ipred_dc_left_w16) pop {r4-r5, pc} L(ipred_dc_left_h32): vld1.8 {d0, d1, d2, d3}, [r2, :128] vaddl.u8 q0, d0, d1 vaddl.u8 q1, d2, d3 vadd.u16 q0, q0, q1 vadd.u16 d0, d0, d1 vpadd.u16 d0, d0 vpadd.u16 d0, d0 vrshrn.u16 d0, q0, #5 vdup.8 q0, d0[0] bx r3 L(ipred_dc_left_w32): vmov.8 q1, q0 1: vst1.8 {d0, d1, d2, d3}, [r0, :128], r1 vst1.8 {d0, d1, d2, d3}, [r12, :128], r1 subs r4, r4, #4 vst1.8 {d0, d1, d2, d3}, [r0, :128], r1 vst1.8 {d0, d1, d2, d3}, [r12, :128], r1 bgt 1b pop {r4-r5, pc} L(ipred_dc_left_h64): vld1.8 {d0, d1, d2, d3}, [r2, :128]! vld1.8 {d4, d5, d6, d7}, [r2, :128] vaddl.u8 q0, d0, d1 vaddl.u8 q1, d2, d3 vaddl.u8 q2, d4, d5 vaddl.u8 q3, d6, d7 vadd.u16 q0, q0, q1 vadd.u16 q1, q2, q3 vadd.u16 q0, q0, q1 vadd.u16 d0, d0, d1 vpadd.u16 d0, d0 vpadd.u16 d0, d0 vrshrn.u16 d0, q0, #6 vdup.8 q0, d0[0] bx r3 L(ipred_dc_left_w64): vmov.8 q1, q0 sub r1, r1, #32 1: vst1.8 {d0, d1, d2, d3}, [r0, :128]! vst1.8 {d0, d1, d2, d3}, [r12, :128]! vst1.8 {d0, d1, d2, d3}, [r0, :128], r1 vst1.8 {d0, d1, d2, d3}, [r12, :128], r1 subs r4, r4, #4 vst1.8 {d0, d1, d2, d3}, [r0, :128]! vst1.8 {d0, d1, d2, d3}, [r12, :128]! vst1.8 {d0, d1, d2, d3}, [r0, :128], r1 vst1.8 {d0, d1, d2, d3}, [r12, :128], r1 bgt 1b pop {r4-r5, pc} endfunc // void ipred_dc_8bpc_neon(pixel *dst, const ptrdiff_t stride, // const pixel *const topleft, // const int width, const int height, const int a, // const int max_width, const int max_height); function ipred_dc_8bpc_neon, export=1 push {r4-r6, lr} ldr r4, [sp, #16] sub r2, r2, r4 add lr, r3, r4 // width + height clz r3, r3 clz r12, r4 vdup.16 q15, lr // width + height adr r5, L(ipred_dc_tbl) rbit lr, lr // rbit(width + height) sub r3, r3, #20 // 25 leading bits, minus table offset 5 sub r12, r12, #25 clz lr, lr // ctz(width + height) ldr r3, [r5, r3, lsl #2] ldr r12, [r5, r12, lsl #2] neg lr, lr // -ctz(width + height) add r3, r5, r3 add r5, r5, r12 vshr.u16 q15, q15, #1 // (width + height) >> 1 vdup.16 q14, lr // -ctz(width + height) add r12, r0, r1 lsl r1, r1, #1 bx r5 .align 2 L(ipred_dc_tbl): .word L(ipred_dc_h64) - L(ipred_dc_tbl) + CONFIG_THUMB .word L(ipred_dc_h32) - L(ipred_dc_tbl) + CONFIG_THUMB .word L(ipred_dc_h16) - L(ipred_dc_tbl) + CONFIG_THUMB .word L(ipred_dc_h8) - L(ipred_dc_tbl) + CONFIG_THUMB .word L(ipred_dc_h4) - L(ipred_dc_tbl) + CONFIG_THUMB .word L(ipred_dc_w64) - L(ipred_dc_tbl) + CONFIG_THUMB .word L(ipred_dc_w32) - L(ipred_dc_tbl) + CONFIG_THUMB .word L(ipred_dc_w16) - L(ipred_dc_tbl) + CONFIG_THUMB .word L(ipred_dc_w8) - L(ipred_dc_tbl) + CONFIG_THUMB .word L(ipred_dc_w4) - L(ipred_dc_tbl) + CONFIG_THUMB L(ipred_dc_h4): vld1.32 {d0[]}, [r2, :32]! vpaddl.u8 d0, d0 add r2, r2, #1 vpadd.u16 d0, d0 bx r3 L(ipred_dc_w4): vld1.32 {d1[]}, [r2] vadd.s16 d0, d0, d30 vpaddl.u8 d1, d1 vpadd.u16 d1, d1 cmp r4, #4 vadd.s16 d0, d0, d1 vshl.u16 d0, d0, d28 beq 1f // h = 8/16 movw lr, #(0x3334/2) movw r5, #(0x5556/2) cmp r4, #16 it ne movne lr, r5 vdup.16 d30, lr vqdmulh.s16 d0, d0, d30 1: vdup.8 d0, d0[0] 2: vst1.32 {d0[0]}, [r0, :32], r1 vst1.32 {d0[0]}, [r12, :32], r1 subs r4, r4, #4 vst1.32 {d0[0]}, [r0, :32], r1 vst1.32 {d0[0]}, [r12, :32], r1 bgt 2b pop {r4-r6, pc} L(ipred_dc_h8): vld1.8 {d0}, [r2, :64]! vpaddl.u8 d0, d0 vpadd.u16 d0, d0 add r2, r2, #1 vpadd.u16 d0, d0 bx r3 L(ipred_dc_w8): vld1.8 {d2}, [r2] vadd.s16 d0, d0, d30 vpaddl.u8 d2, d2 vpadd.u16 d2, d2 vpadd.u16 d2, d2 cmp r4, #8 vadd.s16 d0, d0, d2 vshl.u16 d0, d0, d28 beq 1f // h = 4/16/32 cmp r4, #32 movw lr, #(0x3334/2) movw r5, #(0x5556/2) it ne movne lr, r5 vdup.16 d24, lr vqdmulh.s16 d0, d0, d24 1: vdup.8 d0, d0[0] 2: vst1.8 {d0}, [r0, :64], r1 vst1.8 {d0}, [r12, :64], r1 subs r4, r4, #4 vst1.8 {d0}, [r0, :64], r1 vst1.8 {d0}, [r12, :64], r1 bgt 2b pop {r4-r6, pc} L(ipred_dc_h16): vld1.8 {d0, d1}, [r2, :128]! vaddl.u8 q0, d0, d1 vadd.u16 d0, d0, d1 vpadd.u16 d0, d0 add r2, r2, #1 vpadd.u16 d0, d0 bx r3 L(ipred_dc_w16): vld1.8 {d2, d3}, [r2] vadd.s16 d0, d0, d30 vaddl.u8 q1, d2, d3 vadd.u16 d2, d2, d3 vpadd.u16 d2, d2 vpadd.u16 d2, d2 cmp r4, #16 vadd.s16 d0, d0, d2 vshl.u16 d0, d0, d28 beq 1f // h = 4/8/32/64 tst r4, #(32+16+8) // 16 added to make a consecutive bitmask movw lr, #(0x3334/2) movw r5, #(0x5556/2) it ne movne lr, r5 vdup.16 d24, lr vqdmulh.s16 d0, d0, d24 1: vdup.8 q0, d0[0] 2: vst1.8 {d0, d1}, [r0, :128], r1 vst1.8 {d0, d1}, [r12, :128], r1 subs r4, r4, #4 vst1.8 {d0, d1}, [r0, :128], r1 vst1.8 {d0, d1}, [r12, :128], r1 bgt 2b pop {r4-r6, pc} L(ipred_dc_h32): vld1.8 {d0, d1, d2, d3}, [r2, :128]! vaddl.u8 q0, d0, d1 vaddl.u8 q1, d2, d3 vadd.u16 q0, q0, q1 vadd.u16 d0, d0, d1 vpadd.u16 d0, d0 add r2, r2, #1 vpadd.u16 d0, d0 bx r3 L(ipred_dc_w32): vld1.8 {d2, d3, d4, d5}, [r2] vadd.s16 d0, d0, d30 vaddl.u8 q1, d2, d3 vaddl.u8 q2, d4, d5 vadd.u16 q1, q1, q2 vadd.u16 d2, d2, d3 vpadd.u16 d2, d2 vpadd.u16 d2, d2 cmp r4, #32 vadd.s16 d0, d0, d2 vshl.u16 d4, d0, d28 beq 1f // h = 8/16/64 cmp r4, #8 movw lr, #(0x3334/2) movw r5, #(0x5556/2) it ne movne lr, r5 vdup.16 d24, lr vqdmulh.s16 d4, d4, d24 1: vdup.8 q0, d4[0] vdup.8 q1, d4[0] 2: vst1.8 {d0, d1, d2, d3}, [r0, :128], r1 vst1.8 {d0, d1, d2, d3}, [r12, :128], r1 subs r4, r4, #4 vst1.8 {d0, d1, d2, d3}, [r0, :128], r1 vst1.8 {d0, d1, d2, d3}, [r12, :128], r1 bgt 2b pop {r4-r6, pc} L(ipred_dc_h64): vld1.8 {d0, d1, d2, d3}, [r2, :128]! vaddl.u8 q0, d0, d1 vld1.8 {d4, d5, d6, d7}, [r2, :128]! vaddl.u8 q1, d2, d3 vaddl.u8 q2, d4, d5 vaddl.u8 q3, d6, d7 vadd.u16 q0, q0, q1 vadd.u16 q1, q2, q3 vadd.u16 q0, q0, q1 vadd.u16 d0, d0, d1 vpadd.u16 d0, d0 add r2, r2, #1 vpadd.u16 d0, d0 bx r3 L(ipred_dc_w64): vld1.8 {d2, d3, d4, d5}, [r2]! vadd.s16 d0, d0, d30 vaddl.u8 q2, d4, d5 vaddl.u8 q1, d2, d3 vadd.u16 d4, d4, d5 vadd.u16 d2, d2, d3 vld1.8 {d16, d17, d18, d19}, [r2] vpadd.u16 d4, d4 vpadd.u16 d2, d2 vpadd.u16 d4, d4 vpadd.u16 d2, d2 vaddl.u8 q8, d16, d17 vaddl.u8 q9, d18, d19 vadd.u16 d16, d16, d17 vadd.u16 d18, d18, d19 vpadd.u16 d16, d16 vpadd.u16 d18, d18 vpadd.u16 d16, d16 vpadd.u16 d18, d18 vadd.u16 d2, d2, d4 vadd.u16 d3, d16, d18 cmp r4, #64 vadd.s16 d0, d0, d2 vadd.s16 d0, d0, d3 vshl.u16 d18, d0, d28 beq 1f // h = 16/32 movw lr, #(0x5556/2) movt lr, #(0x3334/2) and r5, r4, #31 lsr lr, lr, r5 vdup.16 d30, lr vqdmulh.s16 d18, d18, d30 1: sub r1, r1, #32 vdup.8 q0, d18[0] vdup.8 q1, d18[0] 2: vst1.8 {d0, d1, d2, d3}, [r0, :128]! vst1.8 {d0, d1, d2, d3}, [r12, :128]! vst1.8 {d0, d1, d2, d3}, [r0, :128], r1 vst1.8 {d0, d1, d2, d3}, [r12, :128], r1 subs r4, r4, #4 vst1.8 {d0, d1, d2, d3}, [r0, :128]! vst1.8 {d0, d1, d2, d3}, [r12, :128]! vst1.8 {d0, d1, d2, d3}, [r0, :128], r1 vst1.8 {d0, d1, d2, d3}, [r12, :128], r1 bgt 2b pop {r4-r6, pc} endfunc // void ipred_paeth_8bpc_neon(pixel *dst, const ptrdiff_t stride, // const pixel *const topleft, // const int width, const int height, const int a, // const int max_width, const int max_height); function ipred_paeth_8bpc_neon, export=1 push {r4-r8, lr} ldr r4, [sp, #24] clz lr, r3 adr r5, L(ipred_paeth_tbl) sub lr, lr, #25 ldr lr, [r5, lr, lsl #2] vld1.8 {d4[], d5[]}, [r2] add r8, r2, #1 sub r2, r2, #4 add r5, r5, lr mov r7, #-4 add r6, r0, r1 lsl r1, r1, #1 bx r5 .align 2 L(ipred_paeth_tbl): .word 640f - L(ipred_paeth_tbl) + CONFIG_THUMB .word 320f - L(ipred_paeth_tbl) + CONFIG_THUMB .word 160f - L(ipred_paeth_tbl) + CONFIG_THUMB .word 80f - L(ipred_paeth_tbl) + CONFIG_THUMB .word 40f - L(ipred_paeth_tbl) + CONFIG_THUMB 40: vld1.32 {d6[], d7[]}, [r8] vsubl.u8 q8, d6, d4 // top - topleft 4: vld4.8 {d0[], d1[], d2[], d3[]}, [r2, :32], r7 vzip.32 d0, d1 vzip.32 d2, d3 vaddw.u8 q9, q8, d0 vaddw.u8 q10, q8, d2 vqmovun.s16 d18, q9 // base vqmovun.s16 d19, q10 vmov d1, d2 vabd.u8 q10, q3, q9 // tdiff vabd.u8 q11, q2, q9 // tldiff vabd.u8 q9, q0, q9 // ldiff vmin.u8 q12, q10, q11 // min(tdiff, tldiff) vcge.u8 q10, q11, q10 // tldiff >= tdiff vcge.u8 q9, q12, q9 // min(tdiff, tldiff) >= ldiff vbsl q10, q3, q2 // tdiff <= tldiff ? top : topleft vbit q10, q0, q9 // ldiff <= min ? left : ... vst1.32 {d21[1]}, [r0, :32], r1 vst1.32 {d21[0]}, [r6, :32], r1 subs r4, r4, #4 vst1.32 {d20[1]}, [r0, :32], r1 vst1.32 {d20[0]}, [r6, :32], r1 bgt 4b pop {r4-r8, pc} 80: vld1.8 {d6}, [r8] vsubl.u8 q8, d6, d4 // top - topleft vmov d7, d6 8: vld4.8 {d0[], d1[], d2[], d3[]}, [r2, :32], r7 vaddw.u8 q9, q8, d0 vaddw.u8 q10, q8, d1 vaddw.u8 q11, q8, d2 vaddw.u8 q12, q8, d3 vqmovun.s16 d18, q9 // base vqmovun.s16 d19, q10 vqmovun.s16 d20, q11 vqmovun.s16 d21, q12 vabd.u8 q11, q3, q9 // tdiff vabd.u8 q12, q3, q10 vabd.u8 q13, q2, q9 // tldiff vabd.u8 q14, q2, q10 vabd.u8 q10, q1, q10 // ldiff vabd.u8 q9, q0, q9 vmin.u8 q15, q12, q14 // min(tdiff, tldiff) vcge.u8 q12, q14, q12 // tldiff >= tdiff vmin.u8 q14, q11, q13 // min(tdiff, tldiff) vcge.u8 q11, q13, q11 // tldiff >= tdiff vcge.u8 q10, q15, q10 // min(tdiff, tldiff) >= ldiff vcge.u8 q9, q14, q9 vbsl q12, q3, q2 // tdiff <= tldiff ? top : topleft vbsl q11, q3, q2 vbit q12, q1, q10 // ldiff <= min ? left : ... vbit q11, q0, q9 vst1.8 {d25}, [r0, :64], r1 vst1.8 {d24}, [r6, :64], r1 subs r4, r4, #4 vst1.8 {d23}, [r0, :64], r1 vst1.8 {d22}, [r6, :64], r1 bgt 8b pop {r4-r8, pc} 160: 320: 640: vld1.8 {d6}, [r8]! mov r12, r3 // Set up pointers for four rows in parallel; r0, r6, r5, lr add r5, r0, r1 add lr, r6, r1 lsl r1, r1, #1 sub r1, r1, r3 1: vld4.8 {d0[], d1[], d2[], d3[]}, [r2, :32], r7 2: vsubl.u8 q8, d6, d4 // top - topleft vmov d7, d6 vaddw.u8 q9, q8, d0 vaddw.u8 q10, q8, d1 vaddw.u8 q11, q8, d2 vaddw.u8 q12, q8, d3 vqmovun.s16 d18, q9 // base vqmovun.s16 d19, q10 vqmovun.s16 d20, q11 vqmovun.s16 d21, q12 vabd.u8 q11, q3, q9 // tdiff vabd.u8 q12, q3, q10 vabd.u8 q13, q2, q9 // tldiff vabd.u8 q14, q2, q10 vabd.u8 q10, q1, q10 // ldiff vabd.u8 q9, q0, q9 vmin.u8 q15, q12, q14 // min(tdiff, tldiff) vcge.u8 q12, q14, q12 // tldiff >= tdiff vmin.u8 q14, q11, q13 // min(tdiff, tldiff) vcge.u8 q11, q13, q11 // tldiff >= tdiff vcge.u8 q10, q15, q10 // min(tdiff, tldiff) >= ldiff vcge.u8 q9, q14, q9 vbsl q12, q3, q2 // tdiff <= tldiff ? top : topleft vbsl q11, q3, q2 vbit q12, q1, q10 // ldiff <= min ? left : ... vbit q11, q0, q9 subs r3, r3, #8 vst1.8 {d25}, [r0, :64]! vst1.8 {d24}, [r6, :64]! vst1.8 {d23}, [r5, :64]! vst1.8 {d22}, [lr, :64]! ble 8f vld1.8 {d6}, [r8]! b 2b 8: subs r4, r4, #4 ble 9f // End of horizontal loop, move pointers to next four rows sub r8, r8, r12 add r0, r0, r1 add r6, r6, r1 vld1.8 {d6}, [r8]! add r5, r5, r1 add lr, lr, r1 mov r3, r12 b 1b 9: pop {r4-r8, pc} endfunc // void ipred_smooth_8bpc_neon(pixel *dst, const ptrdiff_t stride, // const pixel *const topleft, // const int width, const int height, const int a, // const int max_width, const int max_height); function ipred_smooth_8bpc_neon, export=1 push {r4-r10, lr} ldr r4, [sp, #32] movrel r10, X(sm_weights) add r12, r10, r4 add r10, r10, r3 clz r9, r3 adr r5, L(ipred_smooth_tbl) sub lr, r2, r4 sub r9, r9, #25 ldr r9, [r5, r9, lsl #2] vld1.8 {d4[]}, [lr] // bottom add r8, r2, #1 add r5, r5, r9 add r6, r0, r1 lsl r1, r1, #1 bx r5 .align 2 L(ipred_smooth_tbl): .word 640f - L(ipred_smooth_tbl) + CONFIG_THUMB .word 320f - L(ipred_smooth_tbl) + CONFIG_THUMB .word 160f - L(ipred_smooth_tbl) + CONFIG_THUMB .word 80f - L(ipred_smooth_tbl) + CONFIG_THUMB .word 40f - L(ipred_smooth_tbl) + CONFIG_THUMB 40: vld1.32 {d16[]}, [r8] // top vld1.32 {d18[]}, [r10, :32] // weights_hor sub r2, r2, #4 mov r7, #-4 vdup.8 q3, d16[3] // right vsubl.u8 q8, d16, d4 // top-bottom vmovl.u8 q9, d18 // weights_hor 4: vld4.8 {d0[], d1[], d2[], d3[]}, [r2, :32], r7 // left vld4.8 {d20[], d21[], d22[], d23[]}, [r12, :32]! // weights_ver vshll.i8 q12, d6, #8 // right*256 vshll.i8 q13, d6, #8 vzip.32 d1, d0 // left, flipped vzip.32 d3, d2 vzip.32 d20, d21 // weights_ver vzip.32 d22, d23 vshll.i8 q14, d4, #8 // bottom*256 vshll.i8 q15, d4, #8 vsubl.u8 q0, d1, d6 // left-right vsubl.u8 q1, d3, d6 vmovl.u8 q10, d20 // weights_ver vmovl.u8 q11, d22 vmla.i16 q12, q1, q9 // right*256 + (left-right)*weights_hor vmla.i16 q13, q0, q9 // (left flipped) vmla.i16 q14, q8, q10 // bottom*256 + (top-bottom)*weights_ver vmla.i16 q15, q8, q11 vhadd.u16 q12, q12, q14 vhadd.u16 q13, q13, q15 vrshrn.i16 d24, q12, #8 vrshrn.i16 d25, q13, #8 vst1.32 {d24[0]}, [r0, :32], r1 vst1.32 {d24[1]}, [r6, :32], r1 subs r4, r4, #4 vst1.32 {d25[0]}, [r0, :32], r1 vst1.32 {d25[1]}, [r6, :32], r1 bgt 4b pop {r4-r10, pc} 80: vld1.8 {d16}, [r8] // top vld1.8 {d18}, [r10, :64] // weights_hor sub r2, r2, #2 mov r7, #-2 vdup.8 q3, d16[7] // right vsubl.u8 q8, d16, d4 // top-bottom vmovl.u8 q9, d18 // weights_hor 8: vld2.8 {d0[], d1[]}, [r2, :16], r7 // left vld2.8 {d20[], d22[]}, [r12, :16]! // weights_ver vshll.i8 q12, d6, #8 // right*256 vshll.i8 q13, d6, #8 vshll.i8 q14, d4, #8 // bottom*256 vshll.i8 q15, d4, #8 vsubl.u8 q1, d0, d6 // left-right (left flipped) vsubl.u8 q0, d1, d6 vmovl.u8 q10, d20 // weights_ver vmovl.u8 q11, d22 vmla.i16 q12, q0, q9 // right*256 + (left-right)*weights_hor vmla.i16 q13, q1, q9 vmla.i16 q14, q8, q10 // bottom*256 + (top-bottom)*weights_ver vmla.i16 q15, q8, q11 vhadd.u16 q12, q12, q14 vhadd.u16 q13, q13, q15 vrshrn.i16 d24, q12, #8 vrshrn.i16 d25, q13, #8 subs r4, r4, #2 vst1.8 {d24}, [r0, :64], r1 vst1.8 {d25}, [r6, :64], r1 bgt 8b pop {r4-r10, pc} 160: 320: 640: add lr, r2, r3 sub r2, r2, #2 mov r7, #-2 vld1.8 {d6[], d7[]}, [lr] // right sub r1, r1, r3 mov r9, r3 1: vld2.8 {d0[], d1[]}, [r2, :16], r7 // left vld2.8 {d20[], d22[]}, [r12, :16]! // weights_ver vsubl.u8 q1, d0, d6 // left-right (left flipped) vsubl.u8 q0, d1, d6 vmovl.u8 q10, d20 // weights_ver vmovl.u8 q11, d22 2: vld1.8 {d16}, [r8]! // top vld1.8 {d18}, [r10, :64]! // weights_hor vshll.i8 q12, d6, #8 // right*256 vshll.i8 q13, d6, #8 vmovl.u8 q9, d18 // weights_hor vshll.i8 q14, d4, #8 // bottom*256 vshll.i8 q15, d4, #8 vsubl.u8 q8, d16, d4 // top-bottom vmla.i16 q12, q0, q9 // right*256 + (left-right)*weights_hor vmla.i16 q13, q1, q9 vmla.i16 q14, q8, q10 // bottom*256 + (top-bottom)*weights_ver vmla.i16 q15, q8, q11 vhadd.u16 q12, q12, q14 vhadd.u16 q13, q13, q15 vrshrn.i16 d24, q12, #8 vrshrn.i16 d25, q13, #8 subs r3, r3, #8 vst1.8 {d24}, [r0, :64]! vst1.8 {d25}, [r6, :64]! bgt 2b subs r4, r4, #2 ble 9f sub r8, r8, r9 sub r10, r10, r9 add r0, r0, r1 add r6, r6, r1 mov r3, r9 b 1b 9: pop {r4-r10, pc} endfunc // void ipred_smooth_v_8bpc_neon(pixel *dst, const ptrdiff_t stride, // const pixel *const topleft, // const int width, const int height, const int a, // const int max_width, const int max_height); function ipred_smooth_v_8bpc_neon, export=1 push {r4-r7, lr} ldr r4, [sp, #20] movrel r7, X(sm_weights) add r7, r7, r4 clz lr, r3 adr r5, L(ipred_smooth_v_tbl) sub r12, r2, r4 sub lr, lr, #25 ldr lr, [r5, lr, lsl #2] vld1.8 {d4[]}, [r12] // bottom add r2, r2, #1 add r5, r5, lr add r6, r0, r1 lsl r1, r1, #1 bx r5 .align 2 L(ipred_smooth_v_tbl): .word 640f - L(ipred_smooth_v_tbl) + CONFIG_THUMB .word 320f - L(ipred_smooth_v_tbl) + CONFIG_THUMB .word 160f - L(ipred_smooth_v_tbl) + CONFIG_THUMB .word 80f - L(ipred_smooth_v_tbl) + CONFIG_THUMB .word 40f - L(ipred_smooth_v_tbl) + CONFIG_THUMB 40: vld1.32 {d6[]}, [r2] // top vsubl.u8 q3, d6, d4 // top-bottom 4: vld4.8 {d16[], d17[], d18[], d19[]}, [r7, :32]! // weights_ver vshll.i8 q10, d4, #8 // bottom*256 vshll.i8 q11, d4, #8 vzip.32 d16, d17 // weights_ver vzip.32 d18, d19 vmovl.u8 q8, d16 // weights_ver vmovl.u8 q9, d18 subs r4, r4, #4 vmla.i16 q10, q3, q8 // bottom*256 + (top-bottom)*weights_ver vmla.i16 q11, q3, q9 vrshrn.i16 d20, q10, #8 vrshrn.i16 d21, q11, #8 vst1.32 {d20[0]}, [r0, :32], r1 vst1.32 {d20[1]}, [r6, :32], r1 vst1.32 {d21[0]}, [r0, :32], r1 vst1.32 {d21[1]}, [r6, :32], r1 bgt 4b pop {r4-r7, pc} 80: vld1.8 {d6}, [r2] // top vsubl.u8 q3, d6, d4 // top-bottom 8: vld4.8 {d16[], d18[], d20[], d22[]}, [r7, :32]! // weights_ver vshll.i8 q12, d4, #8 // bottom*256 vshll.i8 q13, d4, #8 vshll.i8 q14, d4, #8 vshll.i8 q15, d4, #8 vmovl.u8 q8, d16 // weights_ver vmovl.u8 q9, d18 vmovl.u8 q10, d20 vmovl.u8 q11, d22 vmla.i16 q12, q3, q8 // bottom*256 + (top-bottom)*weights_ver vmla.i16 q13, q3, q9 vmla.i16 q14, q3, q10 vmla.i16 q15, q3, q11 vrshrn.i16 d24, q12, #8 vrshrn.i16 d25, q13, #8 vrshrn.i16 d26, q14, #8 vrshrn.i16 d27, q15, #8 vst1.8 {d24}, [r0, :64], r1 vst1.8 {d25}, [r6, :64], r1 subs r4, r4, #4 vst1.8 {d26}, [r0, :64], r1 vst1.8 {d27}, [r6, :64], r1 bgt 8b pop {r4-r7, pc} 160: 320: 640: vpush {q4-q7} // Set up pointers for four rows in parallel; r0, r6, r5, lr add r5, r0, r1 add lr, r6, r1 lsl r1, r1, #1 sub r1, r1, r3 mov r12, r3 1: vld4.8 {d8[], d10[], d12[], d14[]}, [r7, :32]! // weights_ver vmovl.u8 q4, d8 // weights_ver vmovl.u8 q5, d10 vmovl.u8 q6, d12 vmovl.u8 q7, d14 2: vld1.8 {q3}, [r2]! // top vshll.i8 q8, d4, #8 // bottom*256 vshll.i8 q9, d4, #8 vshll.i8 q10, d4, #8 vshll.i8 q11, d4, #8 vsubl.u8 q0, d6, d4 // top-bottom vsubl.u8 q1, d7, d4 vshll.i8 q12, d4, #8 vshll.i8 q13, d4, #8 vshll.i8 q14, d4, #8 vshll.i8 q15, d4, #8 vmla.i16 q8, q0, q4 // bottom*256 + (top-bottom)*weights_ver vmla.i16 q9, q1, q4 vmla.i16 q10, q0, q5 vmla.i16 q11, q1, q5 vmla.i16 q12, q0, q6 // bottom*256 + (top-bottom)*weights_ver vmla.i16 q13, q1, q6 vmla.i16 q14, q0, q7 vmla.i16 q15, q1, q7 vrshrn.i16 d16, q8, #8 vrshrn.i16 d17, q9, #8 vrshrn.i16 d18, q10, #8 vrshrn.i16 d19, q11, #8 vrshrn.i16 d20, q12, #8 vrshrn.i16 d21, q13, #8 vrshrn.i16 d22, q14, #8 vrshrn.i16 d23, q15, #8 subs r3, r3, #16 vst1.8 {q8}, [r0, :128]! vst1.8 {q9}, [r6, :128]! vst1.8 {q10}, [r5, :128]! vst1.8 {q11}, [lr, :128]! bgt 2b subs r4, r4, #4 ble 9f sub r2, r2, r12 add r0, r0, r1 add r6, r6, r1 add r5, r5, r1 add lr, lr, r1 mov r3, r12 b 1b 9: vpop {q4-q7} pop {r4-r7, pc} endfunc // void ipred_smooth_h_8bpc_neon(pixel *dst, const ptrdiff_t stride, // const pixel *const topleft, // const int width, const int height, const int a, // const int max_width, const int max_height); function ipred_smooth_h_8bpc_neon, export=1 push {r4-r8, lr} ldr r4, [sp, #24] movrel r8, X(sm_weights) add r8, r8, r3 clz lr, r3 adr r5, L(ipred_smooth_h_tbl) add r12, r2, r3 sub lr, lr, #25 ldr lr, [r5, lr, lsl #2] vld1.8 {d4[]}, [r12] // right add r5, r5, lr add r6, r0, r1 lsl r1, r1, #1 bx r5 .align 2 L(ipred_smooth_h_tbl): .word 640f - L(ipred_smooth_h_tbl) + CONFIG_THUMB .word 320f - L(ipred_smooth_h_tbl) + CONFIG_THUMB .word 160f - L(ipred_smooth_h_tbl) + CONFIG_THUMB .word 80f - L(ipred_smooth_h_tbl) + CONFIG_THUMB .word 40f - L(ipred_smooth_h_tbl) + CONFIG_THUMB 40: vld1.32 {d6[]}, [r8, :32] // weights_hor sub r2, r2, #4 mov r7, #-4 vmovl.u8 q3, d6 // weights_hor 4: vld4.8 {d0[], d1[], d2[], d3[]}, [r2, :32], r7 // left vshll.i8 q8, d4, #8 // right*256 vshll.i8 q9, d4, #8 vzip.32 d3, d2 // left, flipped vzip.32 d1, d0 vsubl.u8 q1, d3, d4 // left-right vsubl.u8 q0, d1, d4 subs r4, r4, #4 vmla.i16 q8, q1, q3 // right*256 + (left-right)*weights_hor vmla.i16 q9, q0, q3 vrshrn.i16 d16, q8, #8 vrshrn.i16 d17, q9, #8 vst1.32 {d16[0]}, [r0, :32], r1 vst1.32 {d16[1]}, [r6, :32], r1 vst1.32 {d17[0]}, [r0, :32], r1 vst1.32 {d17[1]}, [r6, :32], r1 bgt 4b pop {r4-r8, pc} 80: vld1.8 {d6}, [r8, :64] // weights_hor sub r2, r2, #4 mov r7, #-4 vmovl.u8 q3, d6 // weights_hor 8: vld4.8 {d16[], d18[], d20[], d22[]}, [r2, :32], r7 // left vshll.i8 q12, d4, #8 // right*256 vshll.i8 q13, d4, #8 vshll.i8 q14, d4, #8 vshll.i8 q15, d4, #8 vsubl.u8 q11, d22, d4 // left-right vsubl.u8 q10, d20, d4 vsubl.u8 q9, d18, d4 vsubl.u8 q8, d16, d4 vmla.i16 q12, q11, q3 // right*256 + (left-right)*weights_hor vmla.i16 q13, q10, q3 // (left flipped) vmla.i16 q14, q9, q3 vmla.i16 q15, q8, q3 vrshrn.i16 d24, q12, #8 vrshrn.i16 d25, q13, #8 vrshrn.i16 d26, q14, #8 vrshrn.i16 d27, q15, #8 vst1.8 {d24}, [r0, :64], r1 vst1.8 {d25}, [r6, :64], r1 subs r4, r4, #4 vst1.8 {d26}, [r0, :64], r1 vst1.8 {d27}, [r6, :64], r1 bgt 8b pop {r4-r8, pc} 160: 320: 640: vpush {q4-q7} sub r2, r2, #4 mov r7, #-4 // Set up pointers for four rows in parallel; r0, r6, r5, lr add r5, r0, r1 add lr, r6, r1 lsl r1, r1, #1 sub r1, r1, r3 mov r12, r3 1: vld4.8 {d8[], d10[], d12[], d14[]}, [r2, :32], r7 // left vsubl.u8 q4, d8, d4 // left-right vsubl.u8 q5, d10, d4 vsubl.u8 q6, d12, d4 vsubl.u8 q7, d14, d4 2: vld1.8 {q1}, [r8, :128]! // weights_hor vshll.i8 q8, d4, #8 // right*256 vshll.i8 q9, d4, #8 vshll.i8 q10, d4, #8 vshll.i8 q11, d4, #8 vmovl.u8 q0, d2 // weights_hor vmovl.u8 q1, d3 vshll.i8 q12, d4, #8 vshll.i8 q13, d4, #8 vshll.i8 q14, d4, #8 vshll.i8 q15, d4, #8 vmla.i16 q8, q7, q0 // right*256 + (left-right)*weights_hor vmla.i16 q9, q7, q1 // (left flipped) vmla.i16 q10, q6, q0 vmla.i16 q11, q6, q1 vmla.i16 q12, q5, q0 vmla.i16 q13, q5, q1 vmla.i16 q14, q4, q0 vmla.i16 q15, q4, q1 vrshrn.i16 d16, q8, #8 vrshrn.i16 d17, q9, #8 vrshrn.i16 d18, q10, #8 vrshrn.i16 d19, q11, #8 vrshrn.i16 d20, q12, #8 vrshrn.i16 d21, q13, #8 vrshrn.i16 d22, q14, #8 vrshrn.i16 d23, q15, #8 subs r3, r3, #16 vst1.8 {q8}, [r0, :128]! vst1.8 {q9}, [r6, :128]! vst1.8 {q10}, [r5, :128]! vst1.8 {q11}, [lr, :128]! bgt 2b subs r4, r4, #4 ble 9f sub r8, r8, r12 add r0, r0, r1 add r6, r6, r1 add r5, r5, r1 add lr, lr, r1 mov r3, r12 b 1b 9: vpop {q4-q7} pop {r4-r8, pc} endfunc // void ipred_filter_8bpc_neon(pixel *dst, const ptrdiff_t stride, // const pixel *const topleft, // const int width, const int height, const int filt_idx, // const int max_width, const int max_height); function ipred_filter_8bpc_neon, export=1 push {r4-r8, lr} movw r12, #511 ldrd r4, r5, [sp, #24] and r5, r5, r12 // 511 movrel r6, X(filter_intra_taps) lsl r5, r5, #6 add r6, r6, r5 vld1.8 {d20, d21, d22, d23}, [r6, :128]! clz lr, r3 adr r5, L(ipred_filter_tbl) vld1.8 {d27, d28, d29}, [r6, :64] sub lr, lr, #26 ldr lr, [r5, lr, lsl #2] vmovl.s8 q8, d20 vmovl.s8 q9, d21 add r5, r5, lr vmovl.s8 q10, d22 vmovl.s8 q11, d23 add r6, r0, r1 lsl r1, r1, #1 vmovl.s8 q12, d27 vmovl.s8 q13, d28 vmovl.s8 q14, d29 add r8, r2, #1 sub r2, r2, #2 mov r7, #-2 bx r5 .align 2 L(ipred_filter_tbl): .word 320f - L(ipred_filter_tbl) + CONFIG_THUMB .word 160f - L(ipred_filter_tbl) + CONFIG_THUMB .word 80f - L(ipred_filter_tbl) + CONFIG_THUMB .word 40f - L(ipred_filter_tbl) + CONFIG_THUMB 40: vld1.32 {d0[]}, [r8] // top (0-3) vmovl.u8 q0, d0 // top (0-3) 4: vld1.32 {d2[]}, [r2], r7 // left (0-1) + topleft (2) vmul.i16 q2, q9, d0[0] // p1(top[0]) * filter(1) vmla.i16 q2, q10, d0[1] // p2(top[1]) * filter(2) vmla.i16 q2, q11, d0[2] // p3(top[2]) * filter(3) vmovl.u8 q1, d2 // left (0-1) + topleft (2) vmla.i16 q2, q12, d0[3] // p4(top[3]) * filter(4) vmla.i16 q2, q8, d2[2] // p0(topleft) * filter(0) vmla.i16 q2, q13, d2[1] // p5(left[0]) * filter(5) vmla.i16 q2, q14, d2[0] // p6(left[1]) * filter(6) vqrshrun.s16 d4, q2, #4 subs r4, r4, #2 vst1.32 {d4[0]}, [r0, :32], r1 vmovl.u8 q0, d4 vst1.32 {d4[1]}, [r6, :32], r1 vmov d0, d1 // move top from [4-7] to [0-3] bgt 4b pop {r4-r8, pc} 80: vld1.8 {d0}, [r8] // top (0-7) vmovl.u8 q0, d0 // top (0-7) 8: vld1.32 {d2[]}, [r2], r7 // left (0-1) + topleft (2) vmul.i16 q2, q9, d0[0] // p1(top[0]) * filter(1) vmla.i16 q2, q10, d0[1] // p2(top[1]) * filter(2) vmla.i16 q2, q11, d0[2] // p3(top[2]) * filter(3) vmovl.u8 q1, d2 // left (0-1) + topleft (2) vmla.i16 q2, q12, d0[3] // p4(top[3]) * filter(4) vmla.i16 q2, q8, d2[2] // p0(topleft) * filter(0) vmla.i16 q2, q13, d2[1] // p5(left[0]) * filter(5) vmla.i16 q2, q14, d2[0] // p6(left[1]) * filter(6) vmul.i16 q3, q9, d1[0] // p1(top[0]) * filter(1) vmla.i16 q3, q10, d1[1] // p2(top[1]) * filter(2) vmla.i16 q3, q11, d1[2] // p3(top[2]) * filter(3) vqrshrun.s16 d4, q2, #4 vmovl.u8 q1, d4 // first block, in 16 bit vmla.i16 q3, q12, d1[3] // p4(top[3]) * filter(4) vmla.i16 q3, q8, d0[3] // p0(topleft) * filter(0) vmla.i16 q3, q13, d2[3] // p5(left[0]) * filter(5) vmla.i16 q3, q14, d3[3] // p6(left[1]) * filter(6) vqrshrun.s16 d5, q3, #4 vzip.32 d4, d5 subs r4, r4, #2 vst1.8 {d4}, [r0, :64], r1 vmovl.u8 q0, d5 vst1.8 {d5}, [r6, :64], r1 bgt 8b pop {r4-r8, pc} 160: 320: vpush {q4-q5} sub r1, r1, r3 mov lr, r3 1: vld1.32 {d0[]}, [r2], r7 // left (0-1) + topleft (2) vmovl.u8 q0, d0 // left (0-1) + topleft (2) 2: vld1.8 {q2}, [r8]! // top(0-15) vmul.i16 q3, q8, d0[2] // p0(topleft) * filter(0) vmla.i16 q3, q13, d0[1] // p5(left[0]) * filter(5) vmovl.u8 q1, d4 // top(0-7) vmovl.u8 q2, d5 // top(8-15) vmla.i16 q3, q14, d0[0] // p6(left[1]) * filter(6) vmla.i16 q3, q9, d2[0] // p1(top[0]) * filter(1) vmla.i16 q3, q10, d2[1] // p2(top[1]) * filter(2) vmla.i16 q3, q11, d2[2] // p3(top[2]) * filter(3) vmla.i16 q3, q12, d2[3] // p4(top[3]) * filter(4) vmul.i16 q4, q9, d3[0] // p1(top[0]) * filter(1) vmla.i16 q4, q10, d3[1] // p2(top[1]) * filter(2) vmla.i16 q4, q11, d3[2] // p3(top[2]) * filter(3) vqrshrun.s16 d6, q3, #4 vmovl.u8 q0, d6 // first block, in 16 bit vmla.i16 q4, q12, d3[3] // p4(top[3]) * filter(4) vmla.i16 q4, q8, d2[3] // p0(topleft) * filter(0) vmla.i16 q4, q13, d0[3] // p5(left[0]) * filter(5) vmla.i16 q4, q14, d1[3] // p6(left[1]) * filter(6) vmul.i16 q5, q9, d4[0] // p1(top[0]) * filter(1) vmla.i16 q5, q10, d4[1] // p2(top[1]) * filter(2) vmla.i16 q5, q11, d4[2] // p3(top[2]) * filter(3) vqrshrun.s16 d7, q4, #4 vmovl.u8 q0, d7 // second block, in 16 bit vmla.i16 q5, q12, d4[3] // p4(top[3]) * filter(4) vmla.i16 q5, q8, d3[3] // p0(topleft) * filter(0) vmla.i16 q5, q13, d0[3] // p5(left[0]) * filter(5) vmla.i16 q5, q14, d1[3] // p6(left[1]) * filter(6) vmul.i16 q15, q9, d5[0] // p1(top[0]) * filter(1) vmla.i16 q15, q10, d5[1] // p2(top[1]) * filter(2) vmla.i16 q15, q11, d5[2] // p3(top[2]) * filter(3) vqrshrun.s16 d8, q5, #4 vmovl.u8 q0, d8 // third block, in 16 bit vmov.u8 r12, d5[6] vmla.i16 q15, q12, d5[3] // p4(top[3]) * filter(4) vmla.i16 q15, q8, d4[3] // p0(topleft) * filter(0) vmla.i16 q15, q13, d0[3] // p5(left[0]) * filter(5) vmla.i16 q15, q14, d1[3] // p6(left[1]) * filter(6) vmov.8 d0[4], r12 subs r3, r3, #16 vqrshrun.s16 d9, q15, #4 vst4.32 {d6[0], d7[0], d8[0], d9[0]}, [r0, :128]! vst4.32 {d6[1], d7[1], d8[1], d9[1]}, [r6, :128]! ble 8f vmov.u8 r12, d9[7] vmov.8 d0[0], r12 vmov.u8 r12, d9[3] vmov.8 d0[2], r12 b 2b 8: subs r4, r4, #2 ble 9f sub r8, r6, lr add r0, r0, r1 add r6, r6, r1 mov r3, lr b 1b 9: vpop {q4-q5} pop {r4-r8, pc} endfunc // void pal_pred_8bpc_neon(pixel *dst, const ptrdiff_t stride, // const uint16_t *const pal, const uint8_t *idx, // const int w, const int h); function pal_pred_8bpc_neon, export=1 push {r4-r5, lr} ldrd r4, r5, [sp, #12] vld1.16 {q0}, [r2, :128] clz lr, r4 adr r12, L(pal_pred_tbl) sub lr, lr, #25 ldr lr, [r12, lr, lsl #2] vmovn.i16 d0, q0 add r12, r12, lr add r2, r0, r1 bx r12 .align 2 L(pal_pred_tbl): .word 640f - L(pal_pred_tbl) + CONFIG_THUMB .word 320f - L(pal_pred_tbl) + CONFIG_THUMB .word 160f - L(pal_pred_tbl) + CONFIG_THUMB .word 80f - L(pal_pred_tbl) + CONFIG_THUMB .word 40f - L(pal_pred_tbl) + CONFIG_THUMB 40: lsl r1, r1, #1 4: vld1.8 {q1}, [r3, :128]! subs r5, r5, #4 vtbl.8 d2, {d0}, d2 vtbl.8 d3, {d0}, d3 vst1.32 {d2[0]}, [r0, :32], r1 vst1.32 {d2[1]}, [r2, :32], r1 vst1.32 {d3[0]}, [r0, :32], r1 vst1.32 {d3[1]}, [r2, :32], r1 bgt 4b pop {r4-r5, pc} 80: lsl r1, r1, #1 8: vld1.8 {q1, q2}, [r3, :128]! subs r5, r5, #4 vtbl.8 d2, {d0}, d2 vtbl.8 d3, {d0}, d3 vst1.8 {d2}, [r0, :64], r1 vtbl.8 d4, {d0}, d4 vst1.8 {d3}, [r2, :64], r1 vtbl.8 d5, {d0}, d5 vst1.8 {d4}, [r0, :64], r1 vst1.8 {d5}, [r2, :64], r1 bgt 8b pop {r4-r5, pc} 160: lsl r1, r1, #1 16: vld1.8 {q8, q9}, [r3, :128]! subs r5, r5, #4 vld1.8 {q10, q11}, [r3, :128]! vtbl.8 d16, {d0}, d16 vtbl.8 d17, {d0}, d17 vtbl.8 d18, {d0}, d18 vtbl.8 d19, {d0}, d19 vtbl.8 d20, {d0}, d20 vtbl.8 d21, {d0}, d21 vst1.8 {q8}, [r0, :128], r1 vtbl.8 d22, {d0}, d22 vst1.8 {q9}, [r2, :128], r1 vtbl.8 d23, {d0}, d23 vst1.8 {q10}, [r0, :128], r1 vst1.8 {q11}, [r2, :128], r1 bgt 16b pop {r4-r5, pc} 320: lsl r1, r1, #1 32: vld1.8 {q8, q9}, [r3, :128]! subs r5, r5, #2 vld1.8 {q10, q11}, [r3, :128]! vtbl.8 d16, {d0}, d16 vtbl.8 d17, {d0}, d17 vtbl.8 d18, {d0}, d18 vtbl.8 d19, {d0}, d19 vtbl.8 d20, {d0}, d20 vtbl.8 d21, {d0}, d21 vst1.8 {q8, q9}, [r0, :128], r1 vtbl.8 d22, {d0}, d22 vtbl.8 d23, {d0}, d23 vst1.8 {q10, q11}, [r2, :128], r1 bgt 32b pop {r4-r5, pc} 640: sub r1, r1, #32 64: vld1.8 {q8, q9}, [r3, :128]! subs r5, r5, #1 vld1.8 {q10, q11}, [r3, :128]! vtbl.8 d16, {d0}, d16 vtbl.8 d17, {d0}, d17 vtbl.8 d18, {d0}, d18 vtbl.8 d19, {d0}, d19 vtbl.8 d20, {d0}, d20 vtbl.8 d21, {d0}, d21 vst1.8 {q8, q9}, [r0, :128]! vtbl.8 d22, {d0}, d22 vtbl.8 d23, {d0}, d23 vst1.8 {q10, q11}, [r0, :128], r1 bgt 64b pop {r4-r5, pc} endfunc // void ipred_cfl_128_8bpc_neon(pixel *dst, const ptrdiff_t stride, // const pixel *const topleft, // const int width, const int height, // const int16_t *ac, const int alpha); function ipred_cfl_128_8bpc_neon, export=1 push {r4-r8, lr} ldrd r4, r5, [sp, #24] ldr r6, [sp, #32] clz lr, r3 adr r12, L(ipred_cfl_128_tbl) sub lr, lr, #26 ldr lr, [r12, lr, lsl #2] vmov.i16 q0, #128 // dc vdup.i16 q1, r6 // alpha add r12, r12, lr add r6, r0, r1 lsl r1, r1, #1 bx r12 .align 2 L(ipred_cfl_128_tbl): L(ipred_cfl_splat_tbl): .word L(ipred_cfl_splat_w16) - L(ipred_cfl_128_tbl) + CONFIG_THUMB .word L(ipred_cfl_splat_w16) - L(ipred_cfl_128_tbl) + CONFIG_THUMB .word L(ipred_cfl_splat_w8) - L(ipred_cfl_128_tbl) + CONFIG_THUMB .word L(ipred_cfl_splat_w4) - L(ipred_cfl_128_tbl) + CONFIG_THUMB L(ipred_cfl_splat_w4): vld1.16 {q2, q3}, [r5, :128]! vmul.i16 q2, q2, q1 // diff = ac * alpha vmul.i16 q3, q3, q1 vshr.s16 q8, q2, #15 // sign = diff >> 15 vshr.s16 q9, q3, #15 vadd.i16 q2, q2, q8 // diff + sign vadd.i16 q3, q3, q9 vrshr.s16 q2, q2, #6 // (diff + sign + 32) >> 6 = apply_sign() vrshr.s16 q3, q3, #6 vadd.i16 q2, q2, q0 // dc + apply_sign() vadd.i16 q3, q3, q0 vqmovun.s16 d4, q2 // iclip_pixel(dc + apply_sign()) vqmovun.s16 d5, q3 vst1.32 {d4[0]}, [r0, :32], r1 vst1.32 {d4[1]}, [r6, :32], r1 subs r4, r4, #4 vst1.32 {d5[0]}, [r0, :32], r1 vst1.32 {d5[1]}, [r6, :32], r1 bgt L(ipred_cfl_splat_w4) pop {r4-r8, pc} L(ipred_cfl_splat_w8): vld1.16 {q8, q9}, [r5, :128]! vld1.16 {q10, q11}, [r5, :128]! vmul.i16 q8, q8, q1 // diff = ac * alpha vmul.i16 q9, q9, q1 vmul.i16 q10, q10, q1 vmul.i16 q11, q11, q1 vshr.s16 q12, q8, #15 // sign = diff >> 15 vshr.s16 q13, q9, #15 vshr.s16 q14, q10, #15 vshr.s16 q15, q11, #15 vadd.i16 q8, q8, q12 // diff + sign vadd.i16 q9, q9, q13 vadd.i16 q10, q10, q14 vadd.i16 q11, q11, q15 vrshr.s16 q8, q8, #6 // (diff + sign + 32) >> 6 = apply_sign() vrshr.s16 q9, q9, #6 vrshr.s16 q10, q10, #6 vrshr.s16 q11, q11, #6 vadd.i16 q8, q8, q0 // dc + apply_sign() vadd.i16 q9, q9, q0 vadd.i16 q10, q10, q0 vadd.i16 q11, q11, q0 vqmovun.s16 d16, q8 // iclip_pixel(dc + apply_sign()) vqmovun.s16 d17, q9 vqmovun.s16 d18, q10 vqmovun.s16 d19, q11 vst1.8 {d16}, [r0, :64], r1 vst1.8 {d17}, [r6, :64], r1 subs r4, r4, #4 vst1.8 {d18}, [r0, :64], r1 vst1.8 {d19}, [r6, :64], r1 bgt L(ipred_cfl_splat_w8) pop {r4-r8, pc} L(ipred_cfl_splat_w16): add r12, r5, r3, lsl #1 sub r1, r1, r3 mov lr, r3 1: vld1.16 {q8, q9}, [r5, :128]! vmul.i16 q8, q8, q1 // diff = ac * alpha vld1.16 {q10, q11}, [r12, :128]! vmul.i16 q9, q9, q1 vmul.i16 q10, q10, q1 vmul.i16 q11, q11, q1 vshr.s16 q12, q8, #15 // sign = diff >> 15 vshr.s16 q13, q9, #15 vshr.s16 q14, q10, #15 vshr.s16 q15, q11, #15 vadd.i16 q8, q8, q12 // diff + sign vadd.i16 q9, q9, q13 vadd.i16 q10, q10, q14 vadd.i16 q11, q11, q15 vrshr.s16 q8, q8, #6 // (diff + sign + 32) >> 6 = apply_sign() vrshr.s16 q9, q9, #6 vrshr.s16 q10, q10, #6 vrshr.s16 q11, q11, #6 vadd.i16 q8, q8, q0 // dc + apply_sign() vadd.i16 q9, q9, q0 vadd.i16 q10, q10, q0 vadd.i16 q11, q11, q0 vqmovun.s16 d16, q8 // iclip_pixel(dc + apply_sign()) vqmovun.s16 d17, q9 vqmovun.s16 d18, q10 vqmovun.s16 d19, q11 subs r3, r3, #16 vst1.16 {q8}, [r0, :128]! vst1.16 {q9}, [r6, :128]! bgt 1b subs r4, r4, #2 add r5, r5, lr, lsl #1 add r12, r12, lr, lsl #1 add r0, r0, r1 add r6, r6, r1 mov r3, lr bgt 1b pop {r4-r8, pc} endfunc // void ipred_cfl_top_8bpc_neon(pixel *dst, const ptrdiff_t stride, // const pixel *const topleft, // const int width, const int height, // const int16_t *ac, const int alpha); function ipred_cfl_top_8bpc_neon, export=1 push {r4-r8, lr} ldrd r4, r5, [sp, #24] ldr r6, [sp, #32] clz lr, r3 adr r12, L(ipred_cfl_top_tbl) sub lr, lr, #26 ldr lr, [r12, lr, lsl #2] vdup.16 q1, r6 // alpha add r2, r2, #1 add r12, r12, lr add r6, r0, r1 lsl r1, r1, #1 bx r12 .align 2 L(ipred_cfl_top_tbl): .word 32f - L(ipred_cfl_top_tbl) + CONFIG_THUMB .word 16f - L(ipred_cfl_top_tbl) + CONFIG_THUMB .word 8f - L(ipred_cfl_top_tbl) + CONFIG_THUMB .word 4f - L(ipred_cfl_top_tbl) + CONFIG_THUMB 4: vld1.32 {d0[]}, [r2] vpaddl.u8 d0, d0 vpadd.u16 d0, d0 vrshr.u16 d0, d0, #2 vdup.16 q0, d0[0] b L(ipred_cfl_splat_w4) 8: vld1.8 {d0}, [r2] vpaddl.u8 d0, d0 vpadd.u16 d0, d0 vpadd.u16 d0, d0 vrshr.u16 d0, d0, #3 vdup.16 q0, d0[0] b L(ipred_cfl_splat_w8) 16: vld1.8 {q0}, [r2] vaddl.u8 q0, d0, d1 vadd.u16 d0, d0, d1 vpadd.u16 d0, d0 vpadd.u16 d0, d0 vrshr.u16 d0, d0, #4 vdup.16 q0, d0[0] b L(ipred_cfl_splat_w16) 32: vld1.8 {q2, q3}, [r2] vaddl.u8 q2, d4, d5 vaddl.u8 q3, d6, d7 vadd.u16 q0, q2, q3 vadd.u16 d0, d0, d1 vpadd.u16 d0, d0 vpadd.u16 d0, d0 vrshr.u16 d0, d0, #5 vdup.16 q0, d0[0] b L(ipred_cfl_splat_w16) endfunc // void ipred_cfl_left_8bpc_neon(pixel *dst, const ptrdiff_t stride, // const pixel *const topleft, // const int width, const int height, // const int16_t *ac, const int alpha); function ipred_cfl_left_8bpc_neon, export=1 push {r4-r8, lr} ldrd r4, r5, [sp, #24] ldr r6, [sp, #32] sub r2, r2, r4 clz lr, r3 clz r8, r4 adr r12, L(ipred_cfl_splat_tbl) adr r7, L(ipred_cfl_left_tbl) sub lr, lr, #26 sub r8, r8, #26 ldr lr, [r12, lr, lsl #2] ldr r8, [r7, r8, lsl #2] vdup.16 q1, r6 // alpha add r12, r12, lr add r7, r7, r8 add r6, r0, r1 lsl r1, r1, #1 bx r7 .align 2 L(ipred_cfl_left_tbl): .word L(ipred_cfl_left_h32) - L(ipred_cfl_left_tbl) + CONFIG_THUMB .word L(ipred_cfl_left_h16) - L(ipred_cfl_left_tbl) + CONFIG_THUMB .word L(ipred_cfl_left_h8) - L(ipred_cfl_left_tbl) + CONFIG_THUMB .word L(ipred_cfl_left_h4) - L(ipred_cfl_left_tbl) + CONFIG_THUMB L(ipred_cfl_left_h4): vld1.32 {d0[]}, [r2, :32] vpaddl.u8 d0, d0 vpadd.u16 d0, d0 vrshr.u16 d0, d0, #2 vdup.16 q0, d0[0] bx r12 L(ipred_cfl_left_h8): vld1.8 {d0}, [r2, :64] vpaddl.u8 d0, d0 vpadd.u16 d0, d0 vpadd.u16 d0, d0 vrshr.u16 d0, d0, #3 vdup.16 q0, d0[0] bx r12 L(ipred_cfl_left_h16): vld1.8 {q0}, [r2, :128] vaddl.u8 q0, d0, d1 vadd.u16 d0, d0, d1 vpadd.u16 d0, d0 vpadd.u16 d0, d0 vrshr.u16 d0, d0, #4 vdup.16 q0, d0[0] bx r12 L(ipred_cfl_left_h32): vld1.8 {q2, q3}, [r2, :128] vaddl.u8 q2, d4, d5 vaddl.u8 q3, d6, d7 vadd.u16 q0, q2, q3 vadd.u16 d0, d0, d1 vpadd.u16 d0, d0 vpadd.u16 d0, d0 vrshr.u16 d0, d0, #5 vdup.16 q0, d0[0] bx r12 endfunc // void ipred_cfl_8bpc_neon(pixel *dst, const ptrdiff_t stride, // const pixel *const topleft, // const int width, const int height, // const int16_t *ac, const int alpha); function ipred_cfl_8bpc_neon, export=1 push {r4-r8, lr} ldrd r4, r5, [sp, #24] ldr r6, [sp, #32] sub r2, r2, r4 add r8, r3, r4 // width + height vdup.16 q1, r6 // alpha clz lr, r3 clz r6, r4 vdup.16 d16, r8 // width + height adr r7, L(ipred_cfl_tbl) rbit r8, r8 // rbit(width + height) sub lr, lr, #22 // 26 leading bits, minus table offset 4 sub r6, r6, #26 clz r8, r8 // ctz(width + height) ldr lr, [r7, lr, lsl #2] ldr r6, [r7, r6, lsl #2] neg r8, r8 // -ctz(width + height) add r12, r7, lr add r7, r7, r6 vshr.u16 d16, d16, #1 // (width + height) >> 1 vdup.16 d17, r8 // -ctz(width + height) add r6, r0, r1 lsl r1, r1, #1 bx r7 .align 2 L(ipred_cfl_tbl): .word L(ipred_cfl_h32) - L(ipred_cfl_tbl) + CONFIG_THUMB .word L(ipred_cfl_h16) - L(ipred_cfl_tbl) + CONFIG_THUMB .word L(ipred_cfl_h8) - L(ipred_cfl_tbl) + CONFIG_THUMB .word L(ipred_cfl_h4) - L(ipred_cfl_tbl) + CONFIG_THUMB .word L(ipred_cfl_w32) - L(ipred_cfl_tbl) + CONFIG_THUMB .word L(ipred_cfl_w16) - L(ipred_cfl_tbl) + CONFIG_THUMB .word L(ipred_cfl_w8) - L(ipred_cfl_tbl) + CONFIG_THUMB .word L(ipred_cfl_w4) - L(ipred_cfl_tbl) + CONFIG_THUMB L(ipred_cfl_h4): vld1.32 {d0[]}, [r2, :32]! vpaddl.u8 d0, d0 add r2, r2, #1 vpadd.i16 d0, d0 bx r12 L(ipred_cfl_w4): vld1.32 {d1[]}, [r2] vadd.i16 d0, d0, d16 vpaddl.u8 d1, d1 vpadd.u16 d1, d1 cmp r4, #4 vadd.i16 d0, d0, d1 vshl.u16 d0, d0, d17 beq 1f // h = 8/16 movw lr, #(0x3334/2) movw r8, #(0x5556/2) cmp r4, #16 it ne movne lr, r8 vdup.16 d18, lr vqdmulh.s16 d0, d0, d18 1: vdup.16 q0, d0[0] b L(ipred_cfl_splat_w4) L(ipred_cfl_h8): vld1.8 {d0}, [r2, :64]! vpaddl.u8 d0, d0 vpadd.i16 d0, d0 add r2, r2, #1 vpadd.i16 d0, d0 bx r12 L(ipred_cfl_w8): vld1.8 {d1}, [r2] vadd.i16 d0, d0, d16 vpaddl.u8 d1, d1 vpadd.i16 d1, d1 vpadd.i16 d1, d1 cmp r4, #8 vadd.i16 d0, d0, d1 vshl.u16 d0, d0, d17 beq 1f // h = 4/16/32 cmp r4, #32 movw lr, #(0x3334/2) movw r8, #(0x5556/2) it ne movne lr, r8 vdup.16 d18, lr vqdmulh.s16 d0, d0, d18 1: vdup.16 q0, d0[0] b L(ipred_cfl_splat_w8) L(ipred_cfl_h16): vld1.8 {q0}, [r2, :128]! vaddl.u8 q0, d0, d1 vadd.i16 d0, d0, d1 vpadd.i16 d0, d0 add r2, r2, #1 vpadd.i16 d0, d0 bx r12 L(ipred_cfl_w16): vld1.8 {q2}, [r2] vadd.i16 d0, d0, d16 vaddl.u8 q2, d4, d5 vadd.i16 d4, d4, d5 vpadd.i16 d4, d4 vpadd.i16 d4, d4 cmp r4, #16 vadd.i16 d0, d0, d4 vshl.u16 d0, d0, d17 beq 1f // h = 4/8/32/64 tst r4, #(32+16+8) // 16 added to make a consecutive bitmask movw lr, #(0x3334/2) movw r8, #(0x5556/2) it ne movne lr, r8 vdup.16 d18, lr vqdmulh.s16 d0, d0, d18 1: vdup.16 q0, d0[0] b L(ipred_cfl_splat_w16) L(ipred_cfl_h32): vld1.8 {q2, q3}, [r2, :128]! vaddl.u8 q2, d4, d5 vaddl.u8 q3, d6, d7 vadd.i16 q0, q2, q3 vadd.i16 d0, d0, d1 vpadd.i16 d0, d0 add r2, r2, #1 vpadd.i16 d0, d0 bx r12 L(ipred_cfl_w32): vld1.8 {q2, q3}, [r2] vadd.i16 d0, d0, d16 vaddl.u8 q2, d4, d5 vaddl.u8 q3, d6, d7 vadd.i16 q2, q2, q3 vadd.i16 d4, d4, d5 vpadd.i16 d4, d4 vpadd.i16 d4, d4 cmp r4, #32 vadd.i16 d0, d0, d4 vshl.u16 d0, d0, d17 beq 1f // h = 8/16/64 cmp r4, #8 movw lr, #(0x3334/2) movw r8, #(0x5556/2) it ne movne lr, r8 vdup.16 d18, lr vqdmulh.s16 d0, d0, d18 1: vdup.16 q0, d0[0] b L(ipred_cfl_splat_w16) endfunc // void cfl_ac_420_8bpc_neon(int16_t *const ac, const pixel *const ypx, // const ptrdiff_t stride, const int w_pad, // const int h_pad, const int cw, const int ch); function ipred_cfl_ac_420_8bpc_neon, export=1 push {r4-r8,lr} ldrd r4, r5, [sp, #24] ldr r6, [sp, #32] clz r8, r5 lsl r4, r4, #2 adr r7, L(ipred_cfl_ac_420_tbl) sub r8, r8, #27 ldr r8, [r7, r8, lsl #2] vmov.i16 q8, #0 vmov.i16 q9, #0 vmov.i16 q10, #0 vmov.i16 q11, #0 add r7, r7, r8 sub r8, r6, r4 // height - h_pad rbit lr, r5 // rbit(width) rbit r12, r6 // rbit(height) clz lr, lr // ctz(width) clz r12, r12 // ctz(height) add lr, lr, r12 // log2sz add r12, r1, r2 vdup.32 d31, lr lsl r2, r2, #1 vneg.s32 d31, d31 // -log2sz bx r7 .align 2 L(ipred_cfl_ac_420_tbl): .word L(ipred_cfl_ac_420_w16) - L(ipred_cfl_ac_420_tbl) + CONFIG_THUMB .word L(ipred_cfl_ac_420_w8) - L(ipred_cfl_ac_420_tbl) + CONFIG_THUMB .word L(ipred_cfl_ac_420_w4) - L(ipred_cfl_ac_420_tbl) + CONFIG_THUMB L(ipred_cfl_ac_420_w4): 1: // Copy and subsample input vld1.8 {d0}, [r1, :64], r2 vld1.8 {d2}, [r12, :64], r2 vld1.8 {d1}, [r1, :64], r2 vld1.8 {d3}, [r12, :64], r2 vpaddl.u8 q0, q0 vpaddl.u8 q1, q1 vadd.i16 q0, q0, q1 vshl.i16 q0, q0, #1 subs r8, r8, #2 vst1.16 {q0}, [r0, :128]! vadd.i16 q8, q8, q0 bgt 1b cmp r4, #0 vmov d0, d1 vmov d2, d1 vmov d3, d1 L(ipred_cfl_ac_420_w4_hpad): beq 3f // This assumes that all callers already did "cmp r4, #0" 2: // Vertical padding (h_pad > 0) subs r4, r4, #4 vst1.16 {q0, q1}, [r0, :128]! vadd.i16 q8, q8, q0 vadd.i16 q8, q8, q1 bgt 2b 3: L(ipred_cfl_ac_420_w4_calc_subtract_dc): // Aggregate the sums vadd.i16 q0, q8, q9 vadd.i16 q1, q10, q11 vpaddl.u16 q0, q0 vpaddl.u16 q1, q1 vadd.i32 q0, q1 vadd.i32 d0, d0, d1 vpadd.i32 d0, d0, d0 // sum sub r0, r0, r6, lsl #3 vrshl.u32 d16, d0, d31 // (sum + (1 << (log2sz - 1))) >>= log2sz vdup.16 q8, d16[0] L(ipred_cfl_ac_420_w4_subtract_dc): 6: // Subtract dc from ac vld1.16 {q0, q1}, [r0, :128] subs r6, r6, #4 vsub.i16 q0, q0, q8 vsub.i16 q1, q1, q8 vst1.16 {q0, q1}, [r0, :128]! bgt 6b pop {r4-r8, pc} L(ipred_cfl_ac_420_w8): cmp r3, #0 bne L(ipred_cfl_ac_420_w8_wpad) 1: // Copy and subsample input, without padding vld1.8 {q0}, [r1, :128], r2 vld1.8 {q1}, [r12, :128], r2 vld1.8 {q2}, [r1, :128], r2 vpaddl.u8 q0, q0 vld1.8 {q3}, [r12, :128], r2 vpaddl.u8 q1, q1 vpaddl.u8 q2, q2 vpaddl.u8 q3, q3 vadd.i16 q0, q0, q1 vadd.i16 q2, q2, q3 vshl.i16 q0, q0, #1 vshl.i16 q1, q2, #1 subs r8, r8, #2 vst1.16 {q0, q1}, [r0, :128]! vadd.i16 q8, q8, q0 vadd.i16 q9, q9, q1 bgt 1b cmp r4, #0 vmov q0, q1 b L(ipred_cfl_ac_420_w8_hpad) L(ipred_cfl_ac_420_w8_wpad): 1: // Copy and subsample input, padding 4 vld1.16 {d0}, [r1, :64], r2 vld1.16 {d2}, [r12, :64], r2 vld1.16 {d1}, [r1, :64], r2 vld1.16 {d3}, [r12, :64], r2 vpaddl.u8 q0, q0 vpaddl.u8 q1, q1 vadd.i16 q0, q0, q1 vshl.i16 q0, q0, #1 vdup.16 d3, d1[3] vmov d2, d1 vdup.16 d1, d0[3] subs r8, r8, #2 vst1.16 {q0, q1}, [r0, :128]! vadd.i16 q8, q8, q0 vadd.i16 q9, q9, q1 bgt 1b cmp r4, #0 vmov q0, q1 L(ipred_cfl_ac_420_w8_hpad): beq 3f // This assumes that all callers already did "cmp r4, #0" 2: // Vertical padding (h_pad > 0) subs r4, r4, #4 vst1.16 {q0, q1}, [r0, :128]! vadd.i16 q8, q8, q0 vadd.i16 q9, q9, q1 vst1.16 {q0, q1}, [r0, :128]! vadd.i16 q10, q10, q0 vadd.i16 q11, q11, q1 bgt 2b 3: // Double the height and reuse the w4 summing/subtracting lsl r6, r6, #1 b L(ipred_cfl_ac_420_w4_calc_subtract_dc) L(ipred_cfl_ac_420_w16): adr r7, L(ipred_cfl_ac_420_w16_tbl) ldr r3, [r7, r3, lsl #2] add r7, r7, r3 bx r7 .align 2 L(ipred_cfl_ac_420_w16_tbl): .word L(ipred_cfl_ac_420_w16_wpad0) - L(ipred_cfl_ac_420_w16_tbl) + CONFIG_THUMB .word L(ipred_cfl_ac_420_w16_wpad1) - L(ipred_cfl_ac_420_w16_tbl) + CONFIG_THUMB .word L(ipred_cfl_ac_420_w16_wpad2) - L(ipred_cfl_ac_420_w16_tbl) + CONFIG_THUMB .word L(ipred_cfl_ac_420_w16_wpad3) - L(ipred_cfl_ac_420_w16_tbl) + CONFIG_THUMB L(ipred_cfl_ac_420_w16_wpad0): 1: // Copy and subsample input, without padding vld1.8 {q0, q1}, [r1, :128], r2 vld1.8 {q2, q3}, [r12, :128], r2 vpaddl.u8 q0, q0 vld1.8 {q12, q13}, [r1, :128], r2 vpaddl.u8 q1, q1 vpaddl.u8 q2, q2 vpaddl.u8 q3, q3 vadd.i16 q0, q0, q2 vadd.i16 q1, q1, q3 vld1.8 {q2, q3}, [r12, :128], r2 vpaddl.u8 q12, q12 vpaddl.u8 q13, q13 vpaddl.u8 q2, q2 vpaddl.u8 q3, q3 vadd.i16 q12, q12, q2 vadd.i16 q13, q13, q3 vshl.i16 q0, q0, #1 vshl.i16 q1, q1, #1 vshl.i16 q2, q12, #1 vshl.i16 q3, q13, #1 subs r8, r8, #2 vst1.16 {q0, q1}, [r0, :128]! vadd.i16 q8, q8, q0 vadd.i16 q9, q9, q1 vst1.16 {q2, q3}, [r0, :128]! vadd.i16 q10, q10, q2 vadd.i16 q11, q11, q3 bgt 1b cmp r4, #0 vmov q0, q2 vmov q1, q3 b L(ipred_cfl_ac_420_w16_hpad) L(ipred_cfl_ac_420_w16_wpad1): 1: // Copy and subsample input, padding 4 vldr d2, [r1, #16] vld1.8 {q0}, [r1, :128], r2 vldr d6, [r12, #16] vld1.8 {q2}, [r12, :128], r2 vpaddl.u8 d2, d2 vldr d26, [r1, #16] vpaddl.u8 q0, q0 vld1.8 {q12}, [r1, :128], r2 vpaddl.u8 d6, d6 vldr d30, [r12, #16] vpaddl.u8 q2, q2 vld1.8 {q14}, [r12, :128], r2 vpaddl.u8 d26, d26 vpaddl.u8 q12, q12 vpaddl.u8 d30, d30 vpaddl.u8 q14, q14 vadd.i16 d2, d2, d6 vadd.i16 q0, q0, q2 vadd.i16 d26, d26, d30 vadd.i16 q12, q12, q14 vshl.i16 d2, d2, #1 vshl.i16 q0, q0, #1 vshl.i16 d6, d26, #1 vshl.i16 q2, q12, #1 vdup.16 d3, d2[3] vdup.16 d7, d6[3] subs r8, r8, #2 vst1.16 {q0, q1}, [r0, :128]! vadd.i16 q8, q8, q0 vadd.i16 q9, q9, q1 vst1.16 {q2, q3}, [r0, :128]! vadd.i16 q10, q10, q2 vadd.i16 q11, q11, q3 bgt 1b cmp r4, #0 vmov q0, q2 vmov q1, q3 b L(ipred_cfl_ac_420_w16_hpad) L(ipred_cfl_ac_420_w16_wpad2): 1: // Copy and subsample input, padding 8 vld1.8 {q0}, [r1, :128], r2 vld1.8 {q1}, [r12, :128], r2 vld1.8 {q2}, [r1, :128], r2 vpaddl.u8 q0, q0 vld1.8 {q3}, [r12, :128], r2 vpaddl.u8 q1, q1 vpaddl.u8 q2, q2 vpaddl.u8 q3, q3 vadd.i16 q0, q0, q1 vadd.i16 q2, q2, q3 vshl.i16 q0, q0, #1 vshl.i16 q2, q2, #1 vdup.16 q1, d1[3] vdup.16 q3, d5[3] subs r8, r8, #2 vst1.16 {q0, q1}, [r0, :128]! vadd.i16 q8, q8, q0 vadd.i16 q9, q9, q1 vst1.16 {q2, q3}, [r0, :128]! vadd.i16 q10, q10, q2 vadd.i16 q11, q11, q3 bgt 1b cmp r4, #0 vmov q0, q2 vmov q1, q3 b L(ipred_cfl_ac_420_w16_hpad) L(ipred_cfl_ac_420_w16_wpad3): 1: // Copy and subsample input, padding 12 vld1.8 {d0}, [r1, :64], r2 vld1.8 {d1}, [r12, :64], r2 vld1.8 {d4}, [r1, :64], r2 vpaddl.u8 q0, q0 vld1.8 {d5}, [r12, :64], r2 vpaddl.u8 q2, q2 vadd.i16 d0, d0, d1 vadd.i16 d4, d4, d5 vshl.i16 d0, d0, #1 vshl.i16 d4, d4, #1 vdup.16 q1, d0[3] vdup.16 q3, d4[3] vdup.16 d1, d0[3] vdup.16 d5, d4[3] subs r8, r8, #2 vst1.16 {q0, q1}, [r0, :128]! vadd.i16 q8, q8, q0 vadd.i16 q9, q9, q1 vst1.16 {q2, q3}, [r0, :128]! vadd.i16 q10, q10, q2 vadd.i16 q11, q11, q3 bgt 1b cmp r4, #0 vmov q0, q2 vmov q1, q3 b L(ipred_cfl_ac_420_w16_hpad) L(ipred_cfl_ac_420_w16_hpad): beq 3f // This assumes that all callers already did "cmp r4, #0" 2: // Vertical padding (h_pad > 0) subs r4, r4, #2 vst1.16 {q0, q1}, [r0, :128]! vadd.i16 q8, q8, q0 vadd.i16 q9, q9, q1 vst1.16 {q2, q3}, [r0, :128]! vadd.i16 q10, q10, q2 vadd.i16 q11, q11, q3 bgt 2b 3: // Quadruple the height and reuse the w4 summing/subtracting lsl r6, r6, #2 b L(ipred_cfl_ac_420_w4_calc_subtract_dc) endfunc // void cfl_ac_422_8bpc_neon(int16_t *const ac, const pixel *const ypx, // const ptrdiff_t stride, const int w_pad, // const int h_pad, const int cw, const int ch); function ipred_cfl_ac_422_8bpc_neon, export=1 push {r4-r8,lr} ldrd r4, r5, [sp, #24] ldr r6, [sp, #32] clz r8, r5 lsl r4, r4, #2 adr r7, L(ipred_cfl_ac_422_tbl) sub r8, r8, #27 ldr r8, [r7, r8, lsl #2] vmov.i16 q8, #0 vmov.i16 q9, #0 vmov.i16 q10, #0 vmov.i16 q11, #0 add r7, r7, r8 sub r8, r6, r4 // height - h_pad rbit lr, r5 // rbit(width) rbit r12, r6 // rbit(height) clz lr, lr // ctz(width) clz r12, r12 // ctz(height) add lr, lr, r12 // log2sz add r12, r1, r2 vdup.32 d31, lr lsl r2, r2, #1 vneg.s32 d31, d31 // -log2sz bx r7 .align 2 L(ipred_cfl_ac_422_tbl): .word L(ipred_cfl_ac_422_w16) - L(ipred_cfl_ac_422_tbl) + CONFIG_THUMB .word L(ipred_cfl_ac_422_w8) - L(ipred_cfl_ac_422_tbl) + CONFIG_THUMB .word L(ipred_cfl_ac_422_w4) - L(ipred_cfl_ac_422_tbl) + CONFIG_THUMB L(ipred_cfl_ac_422_w4): 1: // Copy and subsample input vld1.8 {d0}, [r1, :64], r2 vld1.8 {d1}, [r12, :64], r2 vld1.8 {d2}, [r1, :64], r2 vld1.8 {d3}, [r12, :64], r2 vpaddl.u8 q0, q0 vpaddl.u8 q1, q1 vshl.i16 q0, q0, #2 vshl.i16 q1, q1, #2 subs r8, r8, #4 vst1.16 {q0, q1}, [r0, :128]! vadd.i16 q8, q8, q0 vadd.i16 q9, q9, q1 bgt 1b cmp r4, #0 vmov d0, d3 vmov d1, d3 vmov d2, d3 b L(ipred_cfl_ac_420_w4_hpad) L(ipred_cfl_ac_422_w8): cmp r3, #0 bne L(ipred_cfl_ac_422_w8_wpad) 1: // Copy and subsample input, without padding vld1.8 {q0}, [r1, :128], r2 vld1.8 {q1}, [r12, :128], r2 vld1.8 {q2}, [r1, :128], r2 vpaddl.u8 q0, q0 vld1.8 {q3}, [r12, :128], r2 vpaddl.u8 q1, q1 vpaddl.u8 q2, q2 vpaddl.u8 q3, q3 vshl.i16 q0, q0, #2 vshl.i16 q1, q1, #2 vshl.i16 q2, q2, #2 vshl.i16 q3, q3, #2 subs r8, r8, #4 vst1.16 {q0, q1}, [r0, :128]! vadd.i16 q8, q8, q0 vadd.i16 q9, q9, q1 vst1.16 {q2, q3}, [r0, :128]! vadd.i16 q10, q10, q2 vadd.i16 q11, q11, q3 bgt 1b cmp r4, #0 vmov q0, q3 vmov q1, q3 b L(ipred_cfl_ac_420_w8_hpad) L(ipred_cfl_ac_422_w8_wpad): 1: // Copy and subsample input, padding 4 vld1.8 {d0}, [r1, :64], r2 vld1.8 {d1}, [r12, :64], r2 vld1.8 {d2}, [r1, :64], r2 vld1.8 {d3}, [r12, :64], r2 vpaddl.u8 q0, q0 vpaddl.u8 q1, q1 vshl.i16 q0, q0, #2 vshl.i16 q1, q1, #2 vdup.16 d7, d3[3] vmov d6, d3 vdup.16 d5, d2[3] vmov d4, d2 vdup.16 d3, d1[3] vmov d2, d1 vdup.16 d1, d0[3] subs r8, r8, #4 vst1.16 {q0, q1}, [r0, :128]! vadd.i16 q8, q8, q0 vadd.i16 q9, q9, q1 vst1.16 {q2, q3}, [r0, :128]! vadd.i16 q10, q10, q2 vadd.i16 q11, q11, q3 bgt 1b cmp r4, #0 vmov q0, q3 vmov q1, q3 b L(ipred_cfl_ac_420_w8_hpad) L(ipred_cfl_ac_422_w16): adr r7, L(ipred_cfl_ac_422_w16_tbl) ldr r3, [r7, r3, lsl #2] add r7, r7, r3 bx r7 .align 2 L(ipred_cfl_ac_422_w16_tbl): .word L(ipred_cfl_ac_422_w16_wpad0) - L(ipred_cfl_ac_422_w16_tbl) + CONFIG_THUMB .word L(ipred_cfl_ac_422_w16_wpad1) - L(ipred_cfl_ac_422_w16_tbl) + CONFIG_THUMB .word L(ipred_cfl_ac_422_w16_wpad2) - L(ipred_cfl_ac_422_w16_tbl) + CONFIG_THUMB .word L(ipred_cfl_ac_422_w16_wpad3) - L(ipred_cfl_ac_422_w16_tbl) + CONFIG_THUMB L(ipred_cfl_ac_422_w16_wpad0): 1: // Copy and subsample input, without padding vld1.8 {q0, q1}, [r1, :128], r2 vld1.8 {q2, q3}, [r12, :128], r2 vpaddl.u8 q0, q0 vpaddl.u8 q1, q1 vpaddl.u8 q2, q2 vpaddl.u8 q3, q3 vshl.i16 q0, q0, #2 vshl.i16 q1, q1, #2 vshl.i16 q2, q2, #2 vshl.i16 q3, q3, #2 subs r8, r8, #2 vst1.16 {q0, q1}, [r0, :128]! vadd.i16 q8, q8, q0 vadd.i16 q9, q9, q1 vst1.16 {q2, q3}, [r0, :128]! vadd.i16 q10, q10, q2 vadd.i16 q11, q11, q3 bgt 1b cmp r4, #0 vmov q0, q2 vmov q1, q3 b L(ipred_cfl_ac_420_w16_hpad) L(ipred_cfl_ac_422_w16_wpad1): 1: // Copy and subsample input, padding 4 vldr d2, [r1, #16] vld1.8 {q0}, [r1, :128], r2 vldr d6, [r12, #16] vld1.8 {q2}, [r12, :128], r2 vpaddl.u8 d2, d2 vpaddl.u8 q0, q0 vpaddl.u8 d6, d6 vpaddl.u8 q2, q2 vshl.i16 d2, d2, #2 vshl.i16 q0, q0, #2 vshl.i16 d6, d6, #2 vshl.i16 q2, q2, #2 vdup.16 d3, d2[3] vdup.16 d7, d6[3] subs r8, r8, #2 vst1.16 {q0, q1}, [r0, :128]! vadd.i16 q8, q8, q0 vadd.i16 q9, q9, q1 vst1.16 {q2, q3}, [r0, :128]! vadd.i16 q10, q10, q2 vadd.i16 q11, q11, q3 bgt 1b cmp r4, #0 vmov q0, q2 vmov q1, q3 b L(ipred_cfl_ac_420_w16_hpad) L(ipred_cfl_ac_422_w16_wpad2): 1: // Copy and subsample input, padding 8 vld1.8 {q0}, [r1, :128], r2 vld1.8 {q2}, [r12, :128], r2 vpaddl.u8 q0, q0 vpaddl.u8 q2, q2 vshl.i16 q0, q0, #2 vshl.i16 q2, q2, #2 vdup.16 q1, d1[3] vdup.16 q3, d5[3] subs r8, r8, #2 vst1.16 {q0, q1}, [r0, :128]! vadd.i16 q8, q8, q0 vadd.i16 q9, q9, q1 vst1.16 {q2, q3}, [r0, :128]! vadd.i16 q10, q10, q2 vadd.i16 q11, q11, q3 bgt 1b cmp r4, #0 vmov q0, q2 vmov q1, q3 b L(ipred_cfl_ac_420_w16_hpad) L(ipred_cfl_ac_422_w16_wpad3): 1: // Copy and subsample input, padding 12 vld1.8 {d0}, [r1, :64], r2 vld1.8 {d1}, [r12, :64], r2 vpaddl.u8 q0, q0 vshl.i16 q0, q0, #2 vdup.16 q3, d1[3] vdup.16 q1, d0[3] vdup.16 d5, d1[3] vmov d4, d1 vdup.16 d1, d0[3] subs r8, r8, #2 vst1.16 {q0, q1}, [r0, :128]! vadd.i16 q8, q8, q0 vadd.i16 q9, q9, q1 vst1.16 {q2, q3}, [r0, :128]! vadd.i16 q10, q10, q2 vadd.i16 q11, q11, q3 bgt 1b cmp r4, #0 vmov q0, q2 vmov q1, q3 b L(ipred_cfl_ac_420_w16_hpad) endfunc // void cfl_ac_444_8bpc_neon(int16_t *const ac, const pixel *const ypx, // const ptrdiff_t stride, const int w_pad, // const int h_pad, const int cw, const int ch); function ipred_cfl_ac_444_8bpc_neon, export=1 push {r4-r8,lr} ldrd r4, r5, [sp, #24] ldr r6, [sp, #32] clz r8, r5 lsl r4, r4, #2 adr r7, L(ipred_cfl_ac_444_tbl) sub r8, r8, #26 ldr r8, [r7, r8, lsl #2] vmov.i16 q8, #0 vmov.i16 q9, #0 vmov.i16 q10, #0 vmov.i16 q11, #0 add r7, r7, r8 sub r8, r6, r4 // height - h_pad rbit lr, r5 // rbit(width) rbit r12, r6 // rbit(height) clz lr, lr // ctz(width) clz r12, r12 // ctz(height) add lr, lr, r12 // log2sz add r12, r1, r2 vdup.32 d31, lr lsl r2, r2, #1 vneg.s32 d31, d31 // -log2sz bx r7 .align 2 L(ipred_cfl_ac_444_tbl): .word L(ipred_cfl_ac_444_w32) - L(ipred_cfl_ac_444_tbl) + CONFIG_THUMB .word L(ipred_cfl_ac_444_w16) - L(ipred_cfl_ac_444_tbl) + CONFIG_THUMB .word L(ipred_cfl_ac_444_w8) - L(ipred_cfl_ac_444_tbl) + CONFIG_THUMB .word L(ipred_cfl_ac_444_w4) - L(ipred_cfl_ac_444_tbl) + CONFIG_THUMB L(ipred_cfl_ac_444_w4): 1: // Copy and expand input vld1.32 {d0[]}, [r1, :32], r2 vld1.32 {d0[1]}, [r12, :32], r2 vld1.32 {d2[]}, [r1, :32], r2 vld1.32 {d2[1]}, [r12, :32], r2 vshll.u8 q0, d0, #3 vshll.u8 q1, d2, #3 subs r8, r8, #4 vst1.16 {q0, q1}, [r0, :128]! vadd.i16 q8, q8, q0 vadd.i16 q9, q9, q1 bgt 1b cmp r4, #0 vmov d0, d3 vmov d1, d3 vmov d2, d3 b L(ipred_cfl_ac_420_w4_hpad) L(ipred_cfl_ac_444_w8): 1: // Copy and expand input vld1.16 {d0}, [r1, :64], r2 vld1.16 {d2}, [r12, :64], r2 vld1.16 {d4}, [r1, :64], r2 vshll.u8 q0, d0, #3 vld1.16 {d6}, [r12, :64], r2 vshll.u8 q1, d2, #3 vshll.u8 q2, d4, #3 vshll.u8 q3, d6, #3 subs r8, r8, #4 vst1.16 {q0, q1}, [r0, :128]! vadd.i16 q8, q8, q0 vadd.i16 q9, q9, q1 vst1.16 {q2, q3}, [r0, :128]! vadd.i16 q10, q10, q2 vadd.i16 q11, q11, q3 bgt 1b cmp r4, #0 vmov q0, q3 vmov q1, q3 b L(ipred_cfl_ac_420_w8_hpad) L(ipred_cfl_ac_444_w16): cmp r3, #0 bne L(ipred_cfl_ac_444_w16_wpad) 1: // Copy and expand input, without padding vld1.8 {q1}, [r1, :128], r2 vld1.8 {q3}, [r12, :128], r2 vshll.u8 q0, d2, #3 vshll.u8 q1, d3, #3 vshll.u8 q2, d6, #3 vshll.u8 q3, d7, #3 subs r8, r8, #2 vst1.16 {q0, q1}, [r0, :128]! vadd.i16 q8, q8, q0 vadd.i16 q9, q9, q1 vst1.16 {q2, q3}, [r0, :128]! vadd.i16 q10, q10, q2 vadd.i16 q11, q11, q3 bgt 1b cmp r4, #0 vmov q0, q2 vmov q1, q3 b L(ipred_cfl_ac_420_w16_hpad) L(ipred_cfl_ac_444_w16_wpad): 1: // Copy and expand input, padding 8 vld1.8 {d0}, [r1, :64], r2 vld1.8 {d4}, [r12, :64], r2 vshll.u8 q0, d0, #3 vshll.u8 q2, d4, #3 vdup.16 q1, d1[3] vdup.16 q3, d5[3] subs r8, r8, #2 vst1.16 {q0, q1}, [r0, :128]! vadd.i16 q8, q8, q0 vadd.i16 q9, q9, q1 vst1.16 {q2, q3}, [r0, :128]! vadd.i16 q10, q10, q2 vadd.i16 q11, q11, q3 bgt 1b cmp r4, #0 vmov q0, q2 vmov q1, q3 b L(ipred_cfl_ac_420_w16_hpad) L(ipred_cfl_ac_444_w32): adr r7, L(ipred_cfl_ac_444_w32_tbl) ldr r3, [r7, r3, lsl #1] // (w3>>1) << 2 add r7, r7, r3 bx r7 .align 2 L(ipred_cfl_ac_444_w32_tbl): .word L(ipred_cfl_ac_444_w32_wpad0) - L(ipred_cfl_ac_444_w32_tbl) + CONFIG_THUMB .word L(ipred_cfl_ac_444_w32_wpad2) - L(ipred_cfl_ac_444_w32_tbl) + CONFIG_THUMB .word L(ipred_cfl_ac_444_w32_wpad4) - L(ipred_cfl_ac_444_w32_tbl) + CONFIG_THUMB .word L(ipred_cfl_ac_444_w32_wpad6) - L(ipred_cfl_ac_444_w32_tbl) + CONFIG_THUMB L(ipred_cfl_ac_444_w32_wpad0): 1: // Copy and expand input, without padding vld1.8 {q2, q3}, [r1, :128], r2 vld1.8 {q13, q14}, [r12, :128], r2 vshll.u8 q0, d4, #3 vshll.u8 q1, d5, #3 vshll.u8 q2, d6, #3 vshll.u8 q3, d7, #3 vshll.u8 q12, d26, #3 vshll.u8 q13, d27, #3 subs r8, r8, #2 vst1.16 {q0, q1}, [r0, :128]! vadd.i16 q8, q8, q0 vadd.i16 q9, q9, q1 vshll.u8 q0, d28, #3 vshll.u8 q1, d29, #3 vst1.16 {q2, q3}, [r0, :128]! vadd.i16 q10, q10, q2 vadd.i16 q11, q11, q3 vst1.16 {q12, q13}, [r0, :128]! vadd.i16 q8, q8, q12 vadd.i16 q9, q9, q13 vst1.16 {q0, q1}, [r0, :128]! vadd.i16 q10, q10, q0 vadd.i16 q11, q11, q1 bgt 1b cmp r4, #0 b L(ipred_cfl_ac_444_w32_hpad) L(ipred_cfl_ac_444_w32_wpad2): 1: // Copy and expand input, padding 8 vldr d4, [r1, #16] vld1.8 {q1}, [r1, :128], r2 vldr d28, [r12, #16] vld1.8 {q13}, [r12, :128], r2 vshll.u8 q2, d4, #3 vshll.u8 q0, d2, #3 vshll.u8 q1, d3, #3 vshll.u8 q12, d26, #3 vshll.u8 q13, d27, #3 vdup.16 q3, d5[3] subs r8, r8, #2 vst1.16 {q0, q1}, [r0, :128]! vadd.i16 q8, q8, q0 vadd.i16 q9, q9, q1 vshll.u8 q0, d28, #3 vst1.16 {q2, q3}, [r0, :128]! vadd.i16 q10, q10, q2 vadd.i16 q11, q11, q3 vdup.16 q1, d1[3] vst1.16 {q12, q13}, [r0, :128]! vadd.i16 q8, q8, q12 vadd.i16 q9, q9, q13 vst1.16 {q0, q1}, [r0, :128]! vadd.i16 q10, q10, q0 vadd.i16 q11, q11, q1 bgt 1b cmp r4, #0 b L(ipred_cfl_ac_444_w32_hpad) L(ipred_cfl_ac_444_w32_wpad4): 1: // Copy and expand input, padding 16 vld1.8 {q1}, [r1, :128], r2 vld1.8 {q13}, [r12, :128], r2 vshll.u8 q0, d2, #3 vshll.u8 q1, d3, #3 vshll.u8 q12, d26, #3 vshll.u8 q13, d27, #3 vdup.16 q2, d3[3] vdup.16 q3, d3[3] subs r8, r8, #2 vst1.16 {q0, q1}, [r0, :128]! vadd.i16 q8, q8, q0 vadd.i16 q9, q9, q1 vdup.16 q0, d27[3] vdup.16 q1, d27[3] vst1.16 {q2, q3}, [r0, :128]! vadd.i16 q10, q10, q2 vadd.i16 q11, q11, q3 vst1.16 {q12, q13}, [r0, :128]! vadd.i16 q8, q8, q12 vadd.i16 q9, q9, q13 vst1.16 {q0, q1}, [r0, :128]! vadd.i16 q10, q10, q0 vadd.i16 q11, q11, q1 bgt 1b cmp r4, #0 b L(ipred_cfl_ac_444_w32_hpad) L(ipred_cfl_ac_444_w32_wpad6): 1: // Copy and expand input, padding 24 vld1.8 {d0}, [r1, :64], r2 vld1.8 {d24}, [r12, :64], r2 vshll.u8 q0, d0, #3 vshll.u8 q12, d24, #3 subs r8, r8, #2 vdup.16 q1, d1[3] vdup.16 q2, d1[3] vdup.16 q3, d1[3] vst1.16 {q0, q1}, [r0, :128]! vadd.i16 q8, q8, q0 vadd.i16 q9, q9, q1 vdup.16 q13, d25[3] vdup.16 q0, d25[3] vdup.16 q1, d25[3] vst1.16 {q2, q3}, [r0, :128]! vadd.i16 q10, q10, q2 vadd.i16 q11, q11, q3 vst1.16 {q12, q13}, [r0, :128]! vadd.i16 q8, q8, q12 vadd.i16 q9, q9, q13 vst1.16 {q0, q1}, [r0, :128]! vadd.i16 q10, q10, q0 vadd.i16 q11, q11, q1 bgt 1b cmp r4, #0 L(ipred_cfl_ac_444_w32_hpad): beq 3f // This assumes that all callers already did "cmp r4, #0" 2: // Vertical padding (h_pad > 0) subs r4, r4, #1 vst1.16 {q12, q13}, [r0, :128]! vadd.i16 q8, q8, q12 vadd.i16 q9, q9, q13 vst1.16 {q0, q1}, [r0, :128]! vadd.i16 q10, q10, q0 vadd.i16 q11, q11, q1 bgt 2b 3: // Multiply the height by eight and reuse the w4 subtracting lsl r6, r6, #3 // Aggregate the sums, with wider intermediates earlier than in // ipred_cfl_ac_420_w8_calc_subtract_dc. vpaddl.u16 q0, q8 vpaddl.u16 q1, q9 vpaddl.u16 q2, q10 vpaddl.u16 q3, q11 vadd.i32 q0, q0, q1 vadd.i32 q2, q2, q3 vadd.i32 q0, q0, q2 vadd.i32 d0, d0, d1 vpadd.i32 d0, d0, d0 // sum sub r0, r0, r6, lsl #3 vrshl.u32 d16, d0, d31 // (sum + (1 << (log2sz - 1))) >>= log2sz vdup.16 q8, d16[0] b L(ipred_cfl_ac_420_w4_subtract_dc) endfunc