diff options
Diffstat (limited to '')
-rw-r--r-- | third_party/dav1d/src/x86/filmgrain_avx2.asm | 2107 |
1 files changed, 2107 insertions, 0 deletions
diff --git a/third_party/dav1d/src/x86/filmgrain_avx2.asm b/third_party/dav1d/src/x86/filmgrain_avx2.asm new file mode 100644 index 0000000000..55445cf593 --- /dev/null +++ b/third_party/dav1d/src/x86/filmgrain_avx2.asm @@ -0,0 +1,2107 @@ +; Copyright © 2019-2022, VideoLAN and dav1d authors +; Copyright © 2019-2022, Two Orioles, LLC +; All rights reserved. +; +; Redistribution and use in source and binary forms, with or without +; modification, are permitted provided that the following conditions are met: +; +; 1. Redistributions of source code must retain the above copyright notice, this +; list of conditions and the following disclaimer. +; +; 2. Redistributions in binary form must reproduce the above copyright notice, +; this list of conditions and the following disclaimer in the documentation +; and/or other materials provided with the distribution. +; +; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +; ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +; WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +; ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +; (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +; ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +; SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +%include "config.asm" +%include "ext/x86/x86inc.asm" +%include "x86/filmgrain_common.asm" + +%if ARCH_X86_64 + +SECTION_RODATA 32 +pb_mask: db 0,128,128, 0,128, 0, 0,128,128, 0, 0,128, 0,128,128, 0 +gen_shufE: db 0, 1, 8, 9, 2, 3, 10, 11, 4, 5, 12, 13, 6, 7, 14, 15 +gen_shufA: db 0, 1, 2, 3, 2, 3, 4, 5, 4, 5, 6, 7, 6, 7, 8, 9 +gen_shufB: db 2, 3, 4, 5, 4, 5, 6, 7, 6, 7, 8, 9, 8, 9, 10, 11 +gen_shufC: db 4, 5, 6, 7, 6, 7, 8, 9, 8, 9, 10, 11, 10, 11, 12, 13 +gen_shufD: db 6, 7, 8, 9, 8, 9, 10, 11, 10, 11, 12, 13, 12, 13, 14, 15 +; note: the order of (some of) the following constants matter +pb_27_17: times 2 db 27, 17 +byte_blend: db 0, 0, 0, -1 +pb_27_17_17_27: db 27, 17, 17, 27, 0, 32, 0, 32 +pb_17_27: times 2 db 17, 27 +pb_1: times 4 db 1 +pb_23_22: db 23, 22, 0, 32, 0, 32, 0, 32 +next_upperbit_mask: dw 0x100B, 0x2016, 0x402C, 0x8058 +pw_seed_xor: times 2 dw 0xb524 + times 2 dw 0x49d8 +fg_min: times 4 db 0 + times 4 db 16 +fg_max: times 4 db 255 + times 4 db 240 + times 4 db 235 +pd_m65536: dd -65536 +pw_8: times 2 dw 8 +pw_1024: times 2 dw 1024 +hmul_bits: dw 32768, 16384, 8192, 4096 +round: dw 2048, 1024, 512 +mul_bits: dw 256, 128, 64, 32, 16 +round_vals: dw 32, 64, 128, 256, 512 +pw_1: dw 1 + +%macro JMP_TABLE 2-* + %1_8bpc_%2_table: + %xdefine %%base %1_8bpc_%2_table + %xdefine %%prefix mangle(private_prefix %+ _%1_8bpc_%2) + %rep %0 - 2 + dd %%prefix %+ .ar%3 - %%base + %rotate 1 + %endrep +%endmacro + +JMP_TABLE generate_grain_y, avx2, 0, 1, 2, 3 +JMP_TABLE generate_grain_uv_420, avx2, 0, 1, 2, 3 +JMP_TABLE generate_grain_uv_422, avx2, 0, 1, 2, 3 +JMP_TABLE generate_grain_uv_444, avx2, 0, 1, 2, 3 + +SECTION .text + +INIT_YMM avx2 +cglobal generate_grain_y_8bpc, 2, 9, 8, buf, fg_data +%define base r4-generate_grain_y_8bpc_avx2_table + lea r4, [generate_grain_y_8bpc_avx2_table] + vpbroadcastw xm0, [fg_dataq+FGData.seed] + mov r6d, [fg_dataq+FGData.grain_scale_shift] + movq xm1, [base+next_upperbit_mask] + movsxd r5, [fg_dataq+FGData.ar_coeff_lag] + movq xm4, [base+mul_bits] + movq xm5, [base+hmul_bits] + mov r7, -73*82 + mova xm6, [base+pb_mask] + sub bufq, r7 + vpbroadcastw xm7, [base+round+r6*2] + lea r6, [gaussian_sequence] + movsxd r5, [r4+r5*4] +.loop: + pand xm2, xm0, xm1 + psrlw xm3, xm2, 10 + por xm2, xm3 ; bits 0xf, 0x1e, 0x3c and 0x78 are set + pmullw xm2, xm4 ; bits 0x0f00 are set + pmulhuw xm0, xm5 + pshufb xm3, xm6, xm2 ; set 15th bit for next 4 seeds + psllq xm2, xm3, 30 + por xm2, xm3 + psllq xm3, xm2, 15 + por xm2, xm0 ; aggregate each bit into next seed's high bit + por xm3, xm2 ; 4 next output seeds + pshuflw xm0, xm3, q3333 + psrlw xm3, 5 + pand xm2, xm0, xm1 + movq r2, xm3 + psrlw xm3, xm2, 10 + por xm2, xm3 + pmullw xm2, xm4 + pmulhuw xm0, xm5 + movzx r3d, r2w + pshufb xm3, xm6, xm2 + psllq xm2, xm3, 30 + por xm2, xm3 + psllq xm3, xm2, 15 + por xm0, xm2 + movd xm2, [r6+r3*2] + rorx r3, r2, 32 + por xm3, xm0 + shr r2d, 16 + pinsrw xm2, [r6+r2*2], 1 + pshuflw xm0, xm3, q3333 + movzx r2d, r3w + psrlw xm3, 5 + pinsrw xm2, [r6+r2*2], 2 + shr r3d, 16 + movq r2, xm3 + pinsrw xm2, [r6+r3*2], 3 + movzx r3d, r2w + pinsrw xm2, [r6+r3*2], 4 + rorx r3, r2, 32 + shr r2d, 16 + pinsrw xm2, [r6+r2*2], 5 + movzx r2d, r3w + pinsrw xm2, [r6+r2*2], 6 + shr r3d, 16 + pinsrw xm2, [r6+r3*2], 7 + pmulhrsw xm2, xm7 + packsswb xm2, xm2 + movq [bufq+r7], xm2 + add r7, 8 + jl .loop + + ; auto-regression code + add r5, r4 + jmp r5 + +.ar1: + DEFINE_ARGS buf, fg_data, cf3, shift, val3, min, max, x, val0 + mov shiftd, [fg_dataq+FGData.ar_coeff_shift] + movsx cf3d, byte [fg_dataq+FGData.ar_coeffs_y+3] + movd xm5, [fg_dataq+FGData.ar_coeffs_y] + mova xm2, [base+gen_shufC] + DEFINE_ARGS buf, h, cf3, shift, val3, min, max, x, val0 + pinsrb xm5, [base+pb_1], 3 + vpbroadcastw xm3, [base+round_vals+shiftq*2-12] ; rnd + pmovsxbw xm5, xm5 + pshufd xm4, xm5, q0000 + pshufd xm5, xm5, q1111 + sub bufq, 82*73-(82*3+79) + mov hd, 70 + mov mind, -128 + mov maxd, 127 +.y_loop_ar1: + mov xq, -76 + movsx val3d, byte [bufq+xq-1] +.x_loop_ar1: + pmovsxbw xm1, [bufq+xq-82-3] + pshufb xm0, xm1, xm2 + punpckhwd xm1, xm3 + pmaddwd xm0, xm4 + pmaddwd xm1, xm5 + paddd xm0, xm1 +.x_loop_ar1_inner: + movd val0d, xm0 + psrldq xm0, 4 + imul val3d, cf3d + add val3d, val0d + movsx val0d, byte [bufq+xq] + sarx val3d, val3d, shiftd + add val3d, val0d + cmp val3d, maxd + cmovns val3d, maxd + cmp val3d, mind + cmovs val3d, mind + mov [bufq+xq], val3b + ; keep val3d in-place as left for next x iteration + inc xq + jz .x_loop_ar1_end + test xb, 3 + jnz .x_loop_ar1_inner + jmp .x_loop_ar1 +.x_loop_ar1_end: + add bufq, 82 + dec hd + jg .y_loop_ar1 +.ar0: + RET + +.ar2: +%if WIN64 + ; xmm6 and xmm7 already saved + %assign xmm_regs_used 16 + %assign stack_size_padded 168 + SUB rsp, stack_size_padded + movaps [rsp+16*2], xmm8 + movaps [rsp+16*3], xmm9 + movaps [rsp+16*4], xmm10 + movaps [rsp+16*5], xmm11 + movaps [rsp+16*6], xmm12 + movaps [rsp+16*7], xmm13 + movaps [rsp+16*8], xmm14 + movaps [rsp+16*9], xmm15 +%endif + DEFINE_ARGS buf, fg_data, h, x + mov r6d, [fg_dataq+FGData.ar_coeff_shift] + pmovsxbw xm7, [fg_dataq+FGData.ar_coeffs_y+0] ; cf0-7 + movd xm9, [fg_dataq+FGData.ar_coeffs_y+8] ; cf8-11 + vpbroadcastd xm10, [base+round_vals-14+r6*2] + movd xm11, [base+byte_blend+1] + pmovsxbw xm9, xm9 + pshufd xm4, xm7, q0000 + mova xm12, [base+gen_shufA] + pshufd xm5, xm7, q3333 + mova xm13, [base+gen_shufB] + pshufd xm6, xm7, q1111 + mova xm14, [base+gen_shufC] + pshufd xm7, xm7, q2222 + mova xm15, [base+gen_shufD] + pshufd xm8, xm9, q0000 + psrld xm10, 16 + pshufd xm9, xm9, q1111 + sub bufq, 82*73-(82*3+79) + mov hd, 70 +.y_loop_ar2: + mov xq, -76 +.x_loop_ar2: + pmovsxbw xm0, [bufq+xq-82*2-2] ; y=-2,x=[-2,+5] + pmovsxbw xm1, [bufq+xq-82*1-2] ; y=-1,x=[-2,+5] + pshufb xm2, xm0, xm12 + pmaddwd xm2, xm4 + pshufb xm3, xm1, xm13 + pmaddwd xm3, xm5 + paddd xm2, xm3 + pshufb xm3, xm0, xm14 + pmaddwd xm3, xm6 + punpckhqdq xm0, xm0 + punpcklwd xm0, xm1 + pmaddwd xm0, xm7 + pshufb xm1, xm15 + pmaddwd xm1, xm8 + paddd xm2, xm10 + paddd xm2, xm3 + paddd xm0, xm1 + paddd xm2, xm0 + movq xm0, [bufq+xq-2] ; y=0,x=[-2,+5] +.x_loop_ar2_inner: + pmovsxbw xm1, xm0 + pmaddwd xm3, xm9, xm1 + psrldq xm1, 4 ; y=0,x=0 + paddd xm3, xm2 + psrldq xm2, 4 ; shift top to next pixel + psrad xm3, [fg_dataq+FGData.ar_coeff_shift] + ; don't packssdw since we only care about one value + paddw xm3, xm1 + packsswb xm3, xm3 + pextrb [bufq+xq], xm3, 0 + pslldq xm3, 2 + vpblendvb xm0, xm3, xm11 + psrldq xm0, 1 + inc xq + jz .x_loop_ar2_end + test xb, 3 + jnz .x_loop_ar2_inner + jmp .x_loop_ar2 +.x_loop_ar2_end: + add bufq, 82 + dec hd + jg .y_loop_ar2 + RET + +INIT_YMM avx2 +.ar3: +%if WIN64 + ; xmm6 and xmm7 already saved + %assign stack_offset 16 + ALLOC_STACK 16*14 + %assign stack_size stack_size - 16*4 + %assign xmm_regs_used 12 + movaps [rsp+16*12], xmm8 + movaps [rsp+16*13], xmm9 + movaps [rsp+16*14], xmm10 + movaps [rsp+16*15], xmm11 +%else + ALLOC_STACK 16*12 +%endif + mov r6d, [fg_dataq+FGData.ar_coeff_shift] + movd xm11, [base+byte_blend] + pmovsxbw m1, [fg_dataq+FGData.ar_coeffs_y+ 0] ; cf0-15 + pmovsxbw xm2, [fg_dataq+FGData.ar_coeffs_y+16] ; cf16-23 + pshufd m0, m1, q0000 + mova [rsp+16* 0], m0 + pshufd m0, m1, q1111 + mova [rsp+16* 2], m0 + pshufd m0, m1, q2222 + mova [rsp+16* 4], m0 + pshufd m1, m1, q3333 + mova [rsp+16* 6], m1 + pshufd xm0, xm2, q0000 + mova [rsp+16* 8], xm0 + pshufd xm0, xm2, q1111 + mova [rsp+16* 9], xm0 + psrldq xm7, xm2, 10 + mova m8, [base+gen_shufA] + pinsrw xm2, [base+pw_1], 5 + mova m9, [base+gen_shufC] + pshufd xm2, xm2, q2222 + movu m10, [base+gen_shufE] + vpbroadcastw xm6, [base+round_vals-12+r6*2] + pinsrw xm7, [base+round_vals+r6*2-10], 3 + mova [rsp+16*10], xm2 + DEFINE_ARGS buf, fg_data, h, x + sub bufq, 82*73-(82*3+79) + mov hd, 70 +.y_loop_ar3: + mov xq, -76 +.x_loop_ar3: + movu xm5, [bufq+xq-82*3-3] ; y=-3,x=[-3,+12] + vinserti128 m5, [bufq+xq-82*2-3], 1 ; y=-2,x=[-3,+12] + movu xm4, [bufq+xq-82*1-3] ; y=-1,x=[-3,+12] + punpcklbw m3, m5, m5 + punpckhwd m5, m4 + psraw m3, 8 + punpcklbw m5, m5 + psraw m5, 8 + punpcklbw xm4, xm4 + psraw xm4, 8 + pshufb m0, m3, m8 + pmaddwd m0, [rsp+16*0] + pshufb m1, m3, m9 + pmaddwd m1, [rsp+16*2] + shufps m2, m3, m5, q1032 + paddd m0, m1 + pshufb m1, m2, m8 + vperm2i128 m3, m4, 0x21 + pmaddwd m1, [rsp+16*4] + shufps xm2, xm3, q1021 + vpblendd m2, m3, 0xf0 + pshufb m2, m10 + paddd m0, m1 + pmaddwd m2, [rsp+16*6] + pshufb xm1, xm4, xm9 + pmaddwd xm1, [rsp+16*8] + shufps xm4, xm5, q1132 + paddd m0, m2 + pshufb xm2, xm4, xm8 + pshufd xm4, xm4, q2121 + pmaddwd xm2, [rsp+16*9] + punpcklwd xm4, xm6 + pmaddwd xm4, [rsp+16*10] + vextracti128 xm3, m0, 1 + paddd xm0, xm1 + movq xm1, [bufq+xq-3] ; y=0,x=[-3,+4] + paddd xm2, xm4 + paddd xm0, xm2 + paddd xm0, xm3 +.x_loop_ar3_inner: + pmovsxbw xm2, xm1 + pmaddwd xm2, xm7 + pshufd xm3, xm2, q1111 + paddd xm2, xm0 ; add top + paddd xm2, xm3 ; left+cur + psrldq xm0, 4 + psrad xm2, [fg_dataq+FGData.ar_coeff_shift] + ; don't packssdw since we only care about one value + packsswb xm2, xm2 + pextrb [bufq+xq], xm2, 0 + pslldq xm2, 3 + vpblendvb xm1, xm2, xm11 + psrldq xm1, 1 + inc xq + jz .x_loop_ar3_end + test xb, 3 + jnz .x_loop_ar3_inner + jmp .x_loop_ar3 +.x_loop_ar3_end: + add bufq, 82 + dec hd + jg .y_loop_ar3 + RET + +%macro GEN_GRAIN_UV_FN 3 ; ss_name, ss_x, ss_y +INIT_XMM avx2 +cglobal generate_grain_uv_%1_8bpc, 4, 10, 16, buf, bufy, fg_data, uv +%define base r4-generate_grain_uv_%1_8bpc_avx2_table + lea r4, [generate_grain_uv_%1_8bpc_avx2_table] + vpbroadcastw xm0, [fg_dataq+FGData.seed] + mov r6d, [fg_dataq+FGData.grain_scale_shift] + movq xm1, [base+next_upperbit_mask] + movq xm4, [base+mul_bits] + movq xm5, [base+hmul_bits] + mova xm6, [base+pb_mask] + vpbroadcastw xm7, [base+round+r6*2] + vpbroadcastd xm2, [base+pw_seed_xor+uvq*4] + pxor xm0, xm2 + lea r6, [gaussian_sequence] +%if %2 + mov r7d, 73-35*%3 + add bufq, 44 +.loop_y: + mov r5, -44 +%else + mov r5, -73*82 + sub bufq, r5 +%endif +.loop: + pand xm2, xm0, xm1 + psrlw xm3, xm2, 10 + por xm2, xm3 ; bits 0xf, 0x1e, 0x3c and 0x78 are set + pmullw xm2, xm4 ; bits 0x0f00 are set + pmulhuw xm0, xm5 + pshufb xm3, xm6, xm2 ; set 15th bit for next 4 seeds + psllq xm2, xm3, 30 + por xm2, xm3 + psllq xm3, xm2, 15 + por xm2, xm0 ; aggregate each bit into next seed's high bit + por xm2, xm3 ; 4 next output seeds + pshuflw xm0, xm2, q3333 + psrlw xm2, 5 + movq r8, xm2 + movzx r9d, r8w + movd xm2, [r6+r9*2] + rorx r9, r8, 32 + shr r8d, 16 + pinsrw xm2, [r6+r8*2], 1 + movzx r8d, r9w + pinsrw xm2, [r6+r8*2], 2 + shr r9d, 16 + pinsrw xm2, [r6+r9*2], 3 + pmulhrsw xm2, xm7 + packsswb xm2, xm2 + movd [bufq+r5], xm2 + add r5, 4 + jl .loop +%if %2 + add bufq, 82 + dec r7d + jg .loop_y +%endif + + ; auto-regression code + movsxd r6, [fg_dataq+FGData.ar_coeff_lag] + movsxd r6, [base+generate_grain_uv_%1_8bpc_avx2_table+r6*4] + add r6, r4 + jmp r6 + +INIT_YMM avx2 +.ar0: + DEFINE_ARGS buf, bufy, fg_data, uv, unused, shift + imul uvd, 28 + mov shiftd, [fg_dataq+FGData.ar_coeff_shift] + movd xm2, [fg_dataq+FGData.ar_coeffs_uv+uvq] + movd xm3, [base+hmul_bits+shiftq*2] + DEFINE_ARGS buf, bufy, h + pmovsxbw xm2, xm2 +%if %2 + vpbroadcastd m7, [base+pb_1] + vpbroadcastw m6, [base+hmul_bits+2+%3*2] +%endif + vpbroadcastw m2, xm2 + vpbroadcastw m3, xm3 + pxor m12, m12 +%if %2 + sub bufq, 82*(73-35*%3)+82-(82*3+41) +%else + sub bufq, 82*70-3 +%endif + add bufyq, 3+82*3 + mov hd, 70-35*%3 +.y_loop_ar0: +%if %2 + ; first 32 pixels + movu xm4, [bufyq] + vinserti128 m4, [bufyq+32], 1 +%if %3 + movu xm0, [bufyq+82] + vinserti128 m0, [bufyq+82+32], 1 +%endif + movu xm5, [bufyq+16] + vinserti128 m5, [bufyq+48], 1 +%if %3 + movu xm1, [bufyq+82+16] + vinserti128 m1, [bufyq+82+48], 1 +%endif + pmaddubsw m4, m7, m4 +%if %3 + pmaddubsw m0, m7, m0 +%endif + pmaddubsw m5, m7, m5 +%if %3 + pmaddubsw m1, m7, m1 + paddw m4, m0 + paddw m5, m1 +%endif + pmulhrsw m4, m6 + pmulhrsw m5, m6 +%else + xor r3d, r3d + ; first 32x2 pixels +.x_loop_ar0: + movu m4, [bufyq+r3] + pcmpgtb m0, m12, m4 + punpckhbw m5, m4, m0 + punpcklbw m4, m0 +%endif + pmullw m4, m2 + pmullw m5, m2 + pmulhrsw m4, m3 + pmulhrsw m5, m3 +%if %2 + movu m1, [bufq] +%else + movu m1, [bufq+r3] +%endif + pcmpgtb m8, m12, m1 + punpcklbw m0, m1, m8 + punpckhbw m1, m8 + paddw m0, m4 + paddw m1, m5 + packsswb m0, m1 +%if %2 + movu [bufq], m0 +%else + movu [bufq+r3], m0 + add r3d, 32 + cmp r3d, 64 + jl .x_loop_ar0 +%endif + + ; last 6/12 pixels + movu xm4, [bufyq+32*2] +%if %2 +%if %3 + movu xm5, [bufyq+32*2+82] +%endif + pmaddubsw xm4, xm7, xm4 +%if %3 + pmaddubsw xm5, xm7, xm5 + paddw xm4, xm5 +%endif + movq xm0, [bufq+32] + pmulhrsw xm4, xm6 + pmullw xm4, xm2 + pmulhrsw xm4, xm3 + pcmpgtb xm5, xm12, xm0 + punpcklbw xm5, xm0, xm5 + paddw xm4, xm5 + packsswb xm4, xm4 + pblendw xm0, xm4, xm0, 1000b + movq [bufq+32], xm0 +%else + movu xm0, [bufq+64] + pcmpgtb xm1, xm12, xm4 + punpckhbw xm5, xm4, xm1 + punpcklbw xm4, xm1 + pmullw xm5, xm2 + pmullw xm4, xm2 + vpblendd xm1, xm3, xm12, 0x0c + pmulhrsw xm5, xm1 + pmulhrsw xm4, xm3 + pcmpgtb xm1, xm12, xm0 + punpckhbw xm8, xm0, xm1 + punpcklbw xm0, xm1 + paddw xm5, xm8 + paddw xm0, xm4 + packsswb xm0, xm5 + movu [bufq+64], xm0 +%endif + add bufq, 82 + add bufyq, 82<<%3 + dec hd + jg .y_loop_ar0 + RET + +INIT_XMM avx2 +.ar1: + DEFINE_ARGS buf, bufy, fg_data, uv, val3, cf3, min, max, x, shift + imul uvd, 28 + mov shiftd, [fg_dataq+FGData.ar_coeff_shift] + movsx cf3d, byte [fg_dataq+FGData.ar_coeffs_uv+uvq+3] + movd xm4, [fg_dataq+FGData.ar_coeffs_uv+uvq] + pinsrb xm4, [fg_dataq+FGData.ar_coeffs_uv+uvq+4], 3 + DEFINE_ARGS buf, bufy, h, val0, val3, cf3, min, max, x, shift + pmovsxbw xm4, xm4 + pshufd xm5, xm4, q1111 + pshufd xm4, xm4, q0000 + pmovsxwd xm3, [base+round_vals+shiftq*2-12] ; rnd +%if %2 + vpbroadcastd xm7, [base+pb_1] + vpbroadcastw xm6, [base+hmul_bits+2+%3*2] +%endif + vpbroadcastd xm3, xm3 +%if %2 + sub bufq, 82*(73-35*%3)+44-(82*3+41) +%else + sub bufq, 82*70-(82-3) +%endif + add bufyq, 79+82*3 + mov hd, 70-35*%3 + mov mind, -128 + mov maxd, 127 +.y_loop_ar1: + mov xq, -(76>>%2) + movsx val3d, byte [bufq+xq-1] +.x_loop_ar1: + pmovsxbw xm0, [bufq+xq-82-1] ; top/left +%if %2 + movq xm8, [bufyq+xq*2] +%if %3 + movq xm9, [bufyq+xq*2+82] +%endif +%endif + psrldq xm2, xm0, 2 ; top + psrldq xm1, xm0, 4 ; top/right +%if %2 + pmaddubsw xm8, xm7, xm8 +%if %3 + pmaddubsw xm9, xm7, xm9 + paddw xm8, xm9 +%endif + pmulhrsw xm8, xm6 +%else + pmovsxbw xm8, [bufyq+xq] +%endif + punpcklwd xm0, xm2 + punpcklwd xm1, xm8 + pmaddwd xm0, xm4 + pmaddwd xm1, xm5 + paddd xm0, xm1 + paddd xm0, xm3 +.x_loop_ar1_inner: + movd val0d, xm0 + psrldq xm0, 4 + imul val3d, cf3d + add val3d, val0d + sarx val3d, val3d, shiftd + movsx val0d, byte [bufq+xq] + add val3d, val0d + cmp val3d, maxd + cmovns val3d, maxd + cmp val3d, mind + cmovs val3d, mind + mov byte [bufq+xq], val3b + ; keep val3d in-place as left for next x iteration + inc xq + jz .x_loop_ar1_end + test xq, 3 + jnz .x_loop_ar1_inner + jmp .x_loop_ar1 + +.x_loop_ar1_end: + add bufq, 82 + add bufyq, 82<<%3 + dec hd + jg .y_loop_ar1 + RET + +.ar2: + DEFINE_ARGS buf, bufy, fg_data, uv, unused, shift + mov shiftd, [fg_dataq+FGData.ar_coeff_shift] + imul uvd, 28 + vpbroadcastw xm13, [base+round_vals-12+shiftq*2] + pmovsxbw xm7, [fg_dataq+FGData.ar_coeffs_uv+uvq+0] ; cf0-7 + pmovsxbw xm0, [fg_dataq+FGData.ar_coeffs_uv+uvq+8] ; cf8-12 + pinsrw xm0, [base+pw_1], 5 +%if %2 + vpbroadcastw xm12, [base+hmul_bits+2+%3*2] + vpbroadcastd xm11, [base+pb_1] +%endif + DEFINE_ARGS buf, bufy, fg_data, h, unused, x + pshufd xm4, xm7, q0000 + pshufd xm5, xm7, q3333 + pshufd xm6, xm7, q1111 + pshufd xm7, xm7, q2222 + pshufd xm8, xm0, q0000 + pshufd xm9, xm0, q1111 + pshufd xm10, xm0, q2222 +%if %2 + sub bufq, 82*(73-35*%3)+44-(82*3+41) +%else + sub bufq, 82*70-(82-3) +%endif + add bufyq, 79+82*3 + mov hd, 70-35*%3 +.y_loop_ar2: + mov xq, -(76>>%2) + +.x_loop_ar2: + pmovsxbw xm0, [bufq+xq-82*2-2] ; y=-2,x=[-2,+5] + pmovsxbw xm1, [bufq+xq-82*1-2] ; y=-1,x=[-2,+5] + pshufb xm2, xm0, [base+gen_shufA] + pmaddwd xm2, xm4 + pshufb xm3, xm1, [base+gen_shufB] + pmaddwd xm3, xm5 + paddd xm2, xm3 + pshufb xm3, xm0, [base+gen_shufC] + pmaddwd xm3, xm6 + punpckhqdq xm0, xm0 ; y=-2,x=[+2,+5] + punpcklwd xm0, xm1 + pmaddwd xm0, xm7 + pshufb xm1, [gen_shufD] + pmaddwd xm1, xm8 + paddd xm2, xm3 + paddd xm0, xm1 + paddd xm2, xm0 + +%if %2 + movq xm0, [bufyq+xq*2] +%if %3 + movq xm3, [bufyq+xq*2+82] +%endif + pmaddubsw xm0, xm11, xm0 +%if %3 + pmaddubsw xm3, xm11, xm3 + paddw xm0, xm3 +%endif + pmulhrsw xm0, xm12 +%else + pmovsxbw xm0, [bufyq+xq] +%endif + punpcklwd xm0, xm13 + pmaddwd xm0, xm10 + paddd xm2, xm0 + + movq xm0, [bufq+xq-2] ; y=0,x=[-2,+5] +.x_loop_ar2_inner: + pmovsxbw xm0, xm0 + pmaddwd xm3, xm0, xm9 + psrldq xm0, 2 + paddd xm3, xm2 + psrldq xm2, 4 ; shift top to next pixel + psrad xm3, [fg_dataq+FGData.ar_coeff_shift] + pslldq xm3, 2 + paddw xm3, xm0 + pblendw xm0, xm3, 00000010b + packsswb xm0, xm0 + pextrb [bufq+xq], xm0, 1 + inc xq + jz .x_loop_ar2_end + test xb, 3 + jnz .x_loop_ar2_inner + jmp .x_loop_ar2 + +.x_loop_ar2_end: + add bufq, 82 + add bufyq, 82<<%3 + dec hd + jg .y_loop_ar2 + RET + +INIT_YMM avx2 +.ar3: + DEFINE_ARGS buf, bufy, fg_data, uv, unused, shift + mov shiftd, [fg_dataq+FGData.ar_coeff_shift] + imul uvd, 28 + pmovsxbw m0, [fg_dataq+FGData.ar_coeffs_uv+uvq+ 0] ; cf0-15 + pmovsxbw xm1, [fg_dataq+FGData.ar_coeffs_uv+uvq+16] ; cf16-23 + vpbroadcastb xm2, [fg_dataq+FGData.ar_coeffs_uv+uvq+24] ; cf24 [luma] + movd xm13, [base+round_vals-10+shiftq*2] + vpbroadcastd xm14, [base+round_vals-14+shiftq*2] + pshufd m6, m0, q0000 + pshufd m7, m0, q1111 + pshufd m8, m0, q2222 + pshufd m9, m0, q3333 + pshufd xm10, xm1, q0000 + pshufd xm11, xm1, q1111 + pshufhw xm12, xm1, q0000 + psraw xm2, 8 + palignr xm13, xm1, 10 + punpckhwd xm12, xm2 ; interleave luma cf + psrld xm14, 16 + DEFINE_ARGS buf, bufy, fg_data, h, unused, x +%if %2 + vpbroadcastw xm15, [base+hmul_bits+2+%3*2] + sub bufq, 82*(73-35*%3)+44-(82*3+41) +%else + sub bufq, 82*70-(82-3) +%endif + add bufyq, 79+82*3 + mov hd, 70-35*%3 +.y_loop_ar3: + mov xq, -(76>>%2) +.x_loop_ar3: + vbroadcasti128 m3, [bufq+xq-82*2-3] ; y=-2,x=[-3,+12 + palignr xm1, xm3, [bufq+xq-82*3-9], 6 ; y=-3,x=[-3,+12] + vbroadcasti128 m4, [bufq+xq-82*1-3] ; y=-1,x=[-3,+12] + vpblendd m3, m1, 0x0f + pxor m0, m0 + pcmpgtb m2, m0, m3 + pcmpgtb m0, m4 + punpcklbw m1, m3, m2 + punpckhbw m3, m2 + punpcklbw m2, m4, m0 + punpckhbw xm4, xm0 + pshufb m0, m1, [base+gen_shufA] + pmaddwd m0, m6 + pshufb m5, m1, [base+gen_shufC] + pmaddwd m5, m7 + shufps m1, m3, q1032 + paddd m0, m5 + pshufb m5, m1, [base+gen_shufA] + pmaddwd m5, m8 + shufps xm1, xm3, q2121 + vpblendd m1, m2, 0xf0 + pshufb m1, [base+gen_shufE] + pmaddwd m1, m9 + paddd m0, m5 + pshufb xm3, xm2, [base+gen_shufC] + paddd m0, m1 + pmaddwd xm3, xm10 + palignr xm1, xm4, xm2, 2 + punpckhwd xm1, xm2, xm1 + pmaddwd xm1, xm11 + palignr xm4, xm2, 12 + paddd xm3, xm1 +%if %2 + vpbroadcastd xm5, [base+pb_1] + movq xm1, [bufyq+xq*2] + pmaddubsw xm1, xm5, xm1 +%if %3 + movq xm2, [bufyq+xq*2+82] + pmaddubsw xm5, xm2 + paddw xm1, xm5 +%endif + pmulhrsw xm1, xm15 +%else + pmovsxbw xm1, [bufyq+xq] +%endif + punpcklwd xm4, xm1 + pmaddwd xm4, xm12 + movq xm1, [bufq+xq-3] ; y=0,x=[-3,+4] + vextracti128 xm2, m0, 1 + paddd xm0, xm14 + paddd xm3, xm4 + paddd xm0, xm3 + paddd xm0, xm2 +.x_loop_ar3_inner: + pmovsxbw xm1, xm1 + pmaddwd xm2, xm13, xm1 + pshuflw xm3, xm2, q1032 + paddd xm2, xm0 ; add top + paddd xm2, xm3 ; left+cur + psrldq xm0, 4 + psrad xm2, [fg_dataq+FGData.ar_coeff_shift] + psrldq xm1, 2 + ; don't packssdw, we only care about one value + punpckldq xm2, xm2 + pblendw xm1, xm2, 0100b + packsswb xm1, xm1 + pextrb [bufq+xq], xm1, 2 + inc xq + jz .x_loop_ar3_end + test xb, 3 + jnz .x_loop_ar3_inner + jmp .x_loop_ar3 +.x_loop_ar3_end: + add bufq, 82 + add bufyq, 82<<%3 + dec hd + jg .y_loop_ar3 + RET +%endmacro + +INIT_YMM avx2 +cglobal fgy_32x32xn_8bpc, 6, 13, 15, dst, src, stride, fg_data, w, scaling, \ + grain_lut, h, sby, see, overlap +%define base r9-pd_m65536 + lea r9, [pd_m65536] + mov r6d, [fg_dataq+FGData.scaling_shift] + mov r7d, [fg_dataq+FGData.clip_to_restricted_range] + mov sbyd, sbym + mov overlapd, [fg_dataq+FGData.overlap_flag] + vpbroadcastd m8, [base+pd_m65536] + vpbroadcastw m9, [base+mul_bits+r6*2-14] + vpbroadcastd m10, [base+fg_min+r7*4] + vpbroadcastd m11, [base+fg_max+r7*8] + vpbroadcastd m12, [base+pw_1024] + movq xm13, [base+pb_27_17_17_27] + test sbyd, sbyd + setnz r7b + pxor m7, m7 + test r7b, overlapb + jnz .vertical_overlap + + imul seed, sbyd, (173 << 24) | 37 + add seed, (105 << 24) | 178 + rorx seed, seed, 24 + movzx seed, seew + xor seed, [fg_dataq+FGData.seed] + + DEFINE_ARGS dst, src, stride, src_bak, w, scaling, grain_lut, \ + offx, offy, see, overlap + + lea src_bakq, [srcq+wq] + neg wq + sub dstq, srcq + +.loop_x: + rorx r6, seeq, 1 + or seed, 0xEFF4 + test seeb, seeh + lea seed, [r6+0x8000] + cmovp seed, r6d ; updated seed + + rorx offyd, seed, 8 + rorx offxq, seeq, 12 + and offyd, 0xf + imul offyd, 164 + lea offyd, [offyq+offxq*2+747] ; offy*stride+offx + + DEFINE_ARGS dst, src, stride, src_bak, w, scaling, grain_lut, \ + h, offxy, see, overlap + + mov hd, hm + mov grain_lutq, grain_lutmp +.loop_y: + ; src + mova m2, [srcq] + punpcklbw m0, m2, m7 + punpckhbw m1, m2, m7 + + ; scaling[src] + pandn m4, m8, m0 + mova m6, m8 + vpgatherdd m2, [scalingq+m4-0], m8 + psrld m3, m0, 16 + mova m8, m6 + vpgatherdd m4, [scalingq+m3-2], m6 + pandn m5, m8, m1 + mova m6, m8 + vpgatherdd m3, [scalingq+m5-0], m8 + pblendw m2, m4, 0xaa + psrld m4, m1, 16 + mova m8, m6 + vpgatherdd m5, [scalingq+m4-2], m6 + pblendw m3, m5, 0xaa + + ; grain = grain_lut[offy+y][offx+x] + movu m5, [grain_lutq+offxyq] + punpcklbw m4, m5, m7 + punpckhbw m5, m7 + + ; noise = round2(scaling[src] * grain, scaling_shift) + pmaddubsw m2, m4 + pmaddubsw m3, m5 + pmulhrsw m2, m9 + pmulhrsw m3, m9 + + ; dst = clip_pixel(src, noise) + paddw m0, m2 + paddw m1, m3 + packuswb m0, m1 + pmaxub m0, m10 + pminub m0, m11 + mova [dstq+srcq], m0 + + add srcq, strideq + add grain_lutq, 82 + dec hd + jg .loop_y + + add wq, 32 + jge .end + lea srcq, [src_bakq+wq] + test overlapd, overlapd + jz .loop_x + + ; r8m = sbym + cmp dword r8m, 0 + jne .loop_x_hv_overlap + + ; horizontal overlap (without vertical overlap) +.loop_x_h_overlap: + rorx r6, seeq, 1 + or seed, 0xEFF4 + test seeb, seeh + lea seed, [r6+0x8000] + cmovp seed, r6d ; updated seed + + DEFINE_ARGS dst, src, stride, src_bak, w, scaling, grain_lut, \ + offx, offy, see, left_offxy + + lea left_offxyd, [offyq+32] ; previous column's offy*stride+offx + rorx offyd, seed, 8 + rorx offxq, seeq, 12 + and offyd, 0xf + imul offyd, 164 + lea offyd, [offyq+offxq*2+747] ; offy*stride+offx + + DEFINE_ARGS dst, src, stride, src_bak, w, scaling, grain_lut, \ + h, offxy, see, left_offxy + + mov grain_lutq, grain_lutmp + mov hd, hm +.loop_y_h_overlap: + ; src + mova m2, [srcq] + punpcklbw m0, m2, m7 + punpckhbw m1, m2, m7 + + ; scaling[src] + pandn m4, m8, m0 + mova m6, m8 + vpgatherdd m2, [scalingq+m4-0], m8 + psrld m3, m0, 16 + mova m8, m6 + vpgatherdd m4, [scalingq+m3-2], m6 + pandn m5, m8, m1 + mova m6, m8 + vpgatherdd m3, [scalingq+m5-0], m8 + pblendw m2, m4, 0xaa + psrld m4, m1, 16 + mova m8, m6 + vpgatherdd m5, [scalingq+m4-2], m6 + pblendw m3, m5, 0xaa + + ; grain = grain_lut[offy+y][offx+x] + movu m5, [grain_lutq+offxyq] + movd xm4, [grain_lutq+left_offxyq] + punpcklbw xm4, xm5 + pmaddubsw xm4, xm13, xm4 + pmulhrsw xm4, xm12 + packsswb xm4, xm4 + vpblendd m4, m5, 0xfe + punpckhbw m5, m7 + punpcklbw m4, m7 + + ; noise = round2(scaling[src] * grain, scaling_shift) + pmaddubsw m2, m4 + pmaddubsw m3, m5 + pmulhrsw m2, m9 + pmulhrsw m3, m9 + + ; dst = clip_pixel(src, noise) + paddw m0, m2 + paddw m1, m3 + packuswb m0, m1 + pmaxub m0, m10 + pminub m0, m11 + mova [dstq+srcq], m0 + + add srcq, strideq + add grain_lutq, 82 + dec hd + jg .loop_y_h_overlap + + add wq, 32 + jge .end + lea srcq, [src_bakq+wq] + + ; r8m = sbym + cmp dword r8m, 0 + jne .loop_x_hv_overlap + jmp .loop_x_h_overlap + +.vertical_overlap: + DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, \ + unused, sby, see, overlap + + movzx sbyd, sbyb + imul seed, [fg_dataq+FGData.seed], 0x00010001 + imul r7d, sbyd, 173 * 0x00010001 + imul sbyd, 37 * 0x01000100 + add r7d, (105 << 16) | 188 + add sbyd, (178 << 24) | (141 << 8) + and r7d, 0x00ff00ff + and sbyd, 0xff00ff00 + xor seed, r7d + xor seed, sbyd ; (cur_seed << 16) | top_seed + + DEFINE_ARGS dst, src, stride, src_bak, w, scaling, grain_lut, \ + offx, offy, see, overlap + + lea src_bakq, [srcq+wq] + neg wq + sub dstq, srcq + +.loop_x_v_overlap: + vpbroadcastd m14, [pb_27_17] + + ; we assume from the block above that bits 8-15 of r7d are zero'ed + mov r6d, seed + or seed, 0xeff4eff4 + test seeb, seeh + setp r7b ; parity of top_seed + shr seed, 16 + shl r7d, 16 + test seeb, seeh + setp r7b ; parity of cur_seed + or r6d, 0x00010001 + xor r7d, r6d + rorx seed, r7d, 1 ; updated (cur_seed << 16) | top_seed + + rorx offyd, seed, 8 + rorx offxd, seed, 12 + and offyd, 0xf000f + and offxd, 0xf000f + imul offyd, 164 + ; offxy=offy*stride+offx, (cur_offxy << 16) | top_offxy + lea offyd, [offyq+offxq*2+0x10001*747+32*82] + + DEFINE_ARGS dst, src, stride, src_bak, w, scaling, grain_lut, \ + h, offxy, see, overlap, top_offxy + + mov grain_lutq, grain_lutmp + mov hd, hm + movzx top_offxyd, offxyw + shr offxyd, 16 +.loop_y_v_overlap: + ; src + mova m2, [srcq] + punpcklbw m0, m2, m7 + punpckhbw m1, m2, m7 + + ; scaling[src] + pandn m4, m8, m0 + mova m6, m8 + vpgatherdd m2, [scalingq+m4-0], m8 + psrld m3, m0, 16 + mova m8, m6 + vpgatherdd m4, [scalingq+m3-2], m6 + pandn m5, m8, m1 + mova m6, m8 + vpgatherdd m3, [scalingq+m5-0], m8 + pblendw m2, m4, 0xaa + psrld m4, m1, 16 + mova m8, m6 + vpgatherdd m5, [scalingq+m4-2], m6 + pblendw m3, m5, 0xaa + + ; grain = grain_lut[offy+y][offx+x] + movu m6, [grain_lutq+offxyq] + movu m4, [grain_lutq+top_offxyq] + punpcklbw m5, m4, m6 + punpckhbw m4, m6 + pmaddubsw m5, m14, m5 + pmaddubsw m4, m14, m4 + pmulhrsw m5, m12 + pmulhrsw m4, m12 + packsswb m5, m4 + punpcklbw m4, m5, m7 + punpckhbw m5, m7 + + ; noise = round2(scaling[src] * grain, scaling_shift) + pmaddubsw m2, m4 + pmaddubsw m3, m5 + pmulhrsw m2, m9 + pmulhrsw m3, m9 + + ; dst = clip_pixel(src, noise) + paddw m0, m2 + paddw m1, m3 + packuswb m0, m1 + pmaxub m0, m10 + pminub m0, m11 + mova [dstq+srcq], m0 + + add srcq, strideq + add grain_lutq, 82 + dec hb + jz .end_y_v_overlap + vpbroadcastd m14, [pb_17_27] ; swap weights for second v-overlap line + ; 2 lines get vertical overlap, then fall back to non-overlap code for + ; remaining (up to) 30 lines + add hd, 0x80000000 + jnc .loop_y_v_overlap + jmp .loop_y +.end_y_v_overlap: + add wq, 32 + jge .end + lea srcq, [src_bakq+wq] + + ; since fg_dataq.overlap is guaranteed to be set, we never jump + ; back to .loop_x_v_overlap, and instead always fall-through to + ; h+v overlap +.loop_x_hv_overlap: + vpbroadcastd m14, [pb_27_17] + + ; we assume from the block above that bits 8-15 of r7d are zero'ed + mov r6d, seed + or seed, 0xeff4eff4 + test seeb, seeh + setp r7b ; parity of top_seed + shr seed, 16 + shl r7d, 16 + test seeb, seeh + setp r7b ; parity of cur_seed + or r6d, 0x00010001 + xor r7d, r6d + rorx seed, r7d, 1 ; updated (cur_seed << 16) | top_seed + + DEFINE_ARGS dst, src, stride, src_bak, w, scaling, grain_lut, \ + offx, offy, see, left_offxy, top_offxy, topleft_offxy + + lea topleft_offxyd, [top_offxyq+32] + lea left_offxyd, [offyq+32] + rorx offyd, seed, 8 + rorx offxd, seed, 12 + and offyd, 0xf000f + and offxd, 0xf000f + imul offyd, 164 + ; offxy=offy*stride+offx, (cur_offxy << 16) | top_offxy + lea offyd, [offyq+offxq*2+0x10001*747+32*82] + + DEFINE_ARGS dst, src, stride, src_bak, w, scaling, grain_lut, \ + h, offxy, see, left_offxy, top_offxy, topleft_offxy + + mov grain_lutq, grain_lutmp + mov hd, hm + movzx top_offxyd, offxyw + shr offxyd, 16 +.loop_y_hv_overlap: + ; src + mova m2, [srcq] + punpcklbw m0, m2, m7 + punpckhbw m1, m2, m7 + + ; scaling[src] + pandn m4, m8, m0 + mova m6, m8 + vpgatherdd m2, [scalingq+m4-0], m8 + psrld m3, m0, 16 + mova m8, m6 + vpgatherdd m4, [scalingq+m3-2], m6 + pandn m5, m8, m1 + mova m6, m8 + vpgatherdd m3, [scalingq+m5-0], m8 + pblendw m2, m4, 0xaa + psrld m4, m1, 16 + mova m8, m6 + vpgatherdd m5, [scalingq+m4-2], m6 + pblendw m3, m5, 0xaa + + ; grain = grain_lut[offy+y][offx+x] + movu m6, [grain_lutq+offxyq] + movd xm7, [grain_lutq+left_offxyq] + movu m4, [grain_lutq+top_offxyq] + movd xm5, [grain_lutq+topleft_offxyq] + ; do h interpolation first (so top | top/left -> top, left | cur -> cur) + punpcklbw xm7, xm6 + punpcklbw xm5, xm4 + pmaddubsw xm7, xm13, xm7 + pmaddubsw xm5, xm13, xm5 + pmulhrsw xm7, xm12 + pmulhrsw xm5, xm12 + packsswb xm7, xm7 + packsswb xm5, xm5 + vpblendd m7, m6, 0xfe + vpblendd m5, m4, 0xfe + ; followed by v interpolation (top | cur -> cur) + punpckhbw m4, m6 + punpcklbw m5, m7 + pmaddubsw m4, m14, m4 + pmaddubsw m5, m14, m5 + pmulhrsw m4, m12 + pmulhrsw m5, m12 + pxor m7, m7 + packsswb m5, m4 + punpcklbw m4, m5, m7 + punpckhbw m5, m7 + + ; noise = round2(scaling[src] * grain, scaling_shift) + pmaddubsw m2, m4 + pmaddubsw m3, m5 + pmulhrsw m2, m9 + pmulhrsw m3, m9 + + ; dst = clip_pixel(src, noise) + paddw m0, m2 + paddw m1, m3 + packuswb m0, m1 + pmaxub m0, m10 + pminub m0, m11 + mova [dstq+srcq], m0 + + add srcq, strideq + add grain_lutq, 82 + dec hb + jz .end_y_hv_overlap + vpbroadcastd m14, [pb_17_27] ; swap weights for second v-overlap line + ; 2 lines get vertical overlap, then fall back to non-overlap code for + ; remaining (up to) 30 lines + add hd, 0x80000000 + jnc .loop_y_hv_overlap + jmp .loop_y_h_overlap +.end_y_hv_overlap: + add wq, 32 + lea srcq, [src_bakq+wq] + jl .loop_x_hv_overlap +.end: + RET + +%macro FGUV_FN 3 ; name, ss_hor, ss_ver +cglobal fguv_32x32xn_i%1_8bpc, 6, 15, 16, dst, src, stride, fg_data, w, scaling, \ + grain_lut, h, sby, luma, overlap, uv_pl, is_id +%define base r11-pd_m65536 + lea r11, [pd_m65536] + mov r6d, [fg_dataq+FGData.scaling_shift] + mov r7d, [fg_dataq+FGData.clip_to_restricted_range] + mov r9d, is_idm + mov sbyd, sbym + mov overlapd, [fg_dataq+FGData.overlap_flag] + vpbroadcastd m8, [base+pd_m65536] + vpbroadcastw m9, [base+mul_bits+r6*2-14] + vpbroadcastd m10, [base+fg_min+r7*4] + shlx r7d, r7d, r9d + vpbroadcastd m11, [base+fg_max+r7*4] + vpbroadcastd m12, [base+pw_1024] + pxor m7, m7 + test sbyd, sbyd + setnz r7b + cmp byte [fg_dataq+FGData.chroma_scaling_from_luma], 0 + jne .csfl + +%macro %%FGUV_32x32xN_LOOP 3 ; not-csfl, ss_hor, ss_ver + DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, \ + h, sby, see, overlap, uv_pl +%if %1 + mov r6d, uv_plm + vpbroadcastd m0, [base+pw_8] + vbroadcasti128 m14, [fg_dataq+FGData.uv_mult+r6*4] + vpbroadcastw m15, [fg_dataq+FGData.uv_offset+r6*4] + pshufb m14, m0 ; uv_luma_mult, uv_mult +%elif %2 + vpbroadcastq m15, [base+pb_23_22] +%else + vpbroadcastq xm15, [base+pb_27_17_17_27] +%endif +%if %3 + vpbroadcastw m13, [base+pb_23_22] +%elif %2 + pshufd m13, [base+pb_27_17], q0000 ; 8x27_17, 8x17_27 +%endif + test r7b, overlapb + jnz %%vertical_overlap + + imul seed, sbyd, (173 << 24) | 37 + add seed, (105 << 24) | 178 + rorx seed, seed, 24 + movzx seed, seew + xor seed, [fg_dataq+FGData.seed] + + DEFINE_ARGS dst, src, stride, luma, w, scaling, grain_lut, \ + unused2, unused3, see, overlap, unused4, unused5, lstride + + mov lumaq, r9mp + lea r12, [srcq+wq] + lea r13, [dstq+wq] + lea r14, [lumaq+wq*(1+%2)] + mov r11mp, r12 + mov r12mp, r13 + mov lstrideq, r10mp + neg wq + +%%loop_x: + rorx r6, seeq, 1 + or seed, 0xEFF4 + test seeb, seeh + lea seed, [r6+0x8000] + cmovp seed, r6d ; updated seed + + DEFINE_ARGS dst, src, stride, luma, w, scaling, grain_lut, \ + offx, offy, see, overlap, unused1, unused2, lstride + + rorx offyd, seed, 8 + rorx offxq, seeq, 12 + and offyd, 0xf + imul offyd, 164>>%3 + lea offyd, [offyq+offxq*(2-%2)+(3+(6>>%3))*82+3+(6>>%2)] ; offy*stride+offx + + DEFINE_ARGS dst, src, stride, luma, w, scaling, grain_lut, \ + h, offxy, see, overlap, unused1, unused2, lstride + + mov grain_lutq, grain_lutmp + mov hd, hm +%%loop_y: + ; src +%if %2 + mova xm3, [lumaq+lstrideq*0+ 0] + vinserti128 m3, [lumaq+lstrideq*(1+%3) +0], 1 + vpbroadcastd m2, [pb_1] + mova xm0, [lumaq+lstrideq*0+16] + vinserti128 m0, [lumaq+lstrideq*(1+%3)+16], 1 + mova xm1, [srcq] + vinserti128 m1, [srcq+strideq], 1 + pmaddubsw m3, m2 + pmaddubsw m0, m2 + pavgw m3, m7 + pavgw m0, m7 +%else + mova m2, [lumaq] + mova m1, [srcq] +%endif +%if %1 +%if %2 + packuswb m2, m3, m0 ; luma +%endif + punpckhbw m3, m2, m1 + punpcklbw m2, m1 ; { luma, chroma } + pmaddubsw m3, m14 + pmaddubsw m2, m14 + psraw m3, 6 + psraw m2, 6 + paddw m3, m15 + paddw m2, m15 + packuswb m2, m3 ; pack+unpack = clip +%endif +%if %1 || %2 == 0 + punpcklbw m3, m2, m7 + punpckhbw m0, m2, m7 +%endif + + ; scaling[luma_src] + pandn m4, m8, m3 + mova m6, m8 + vpgatherdd m2, [scalingq+m4-0], m8 + psrld m3, 16 + mova m8, m6 + vpgatherdd m4, [scalingq+m3-2], m6 + pandn m5, m8, m0 + mova m6, m8 + vpgatherdd m3, [scalingq+m5-0], m8 + psrld m0, 16 + mova m8, m6 + vpgatherdd m5, [scalingq+m0-2], m6 + pblendw m2, m4, 0xaa + pblendw m3, m5, 0xaa + + ; grain = grain_lut[offy+y][offx+x] +%if %2 + movu xm5, [grain_lutq+offxyq+ 0] + vinserti128 m5, [grain_lutq+offxyq+82], 1 +%else + movu m5, [grain_lutq+offxyq] +%endif + punpcklbw m4, m5, m7 + punpckhbw m5, m7 + + ; noise = round2(scaling[luma_src] * grain, scaling_shift) + pmaddubsw m2, m4 + pmaddubsw m3, m5 + pmulhrsw m2, m9 + pmulhrsw m3, m9 + + ; unpack chroma_source + punpcklbw m0, m1, m7 + punpckhbw m1, m7 + + ; dst = clip_pixel(src, noise) + paddw m0, m2 + paddw m1, m3 + packuswb m0, m1 + pmaxub m0, m10 + pminub m0, m11 +%if %2 + mova [dstq], xm0 + vextracti128 [dstq+strideq], m0, 1 +%else + mova [dstq], m0 +%endif + +%if %2 + lea srcq, [srcq+strideq*2] + lea dstq, [dstq+strideq*2] + lea lumaq, [lumaq+lstrideq*(2<<%3)] +%else + add srcq, strideq + add dstq, strideq + add lumaq, lstrideq +%endif + add grain_lutq, 82<<%2 + sub hb, 1+%2 + jg %%loop_y + + add wq, 32>>%2 + jge .end + mov srcq, r11mp + mov dstq, r12mp + lea lumaq, [r14+wq*(1+%2)] + add srcq, wq + add dstq, wq + test overlapd, overlapd + jz %%loop_x + + ; r8m = sbym + cmp dword r8m, 0 + jne %%loop_x_hv_overlap + + ; horizontal overlap (without vertical overlap) +%%loop_x_h_overlap: + rorx r6, seeq, 1 + or seed, 0xEFF4 + test seeb, seeh + lea seed, [r6+0x8000] + cmovp seed, r6d ; updated seed + + DEFINE_ARGS dst, src, stride, luma, w, scaling, grain_lut, \ + offx, offy, see, left_offxy, unused1, unused2, lstride + + lea left_offxyd, [offyq+(32>>%2)] ; previous column's offy*stride+offx + rorx offyd, seed, 8 + rorx offxq, seeq, 12 + and offyd, 0xf + imul offyd, 164>>%3 + lea offyd, [offyq+offxq*(2-%2)+(3+(6>>%3))*82+3+(6>>%2)] ; offy*stride+offx + + DEFINE_ARGS dst, src, stride, luma, w, scaling, grain_lut, \ + h, offxy, see, left_offxy, unused1, unused2, lstride + + mov grain_lutq, grain_lutmp + mov hd, hm +%%loop_y_h_overlap: + ; src +%if %2 + mova xm3, [lumaq+lstrideq*0+ 0] + vinserti128 m3, [lumaq+lstrideq*(1+%3)+ 0], 1 + vpbroadcastd m2, [pb_1] + mova xm0, [lumaq+lstrideq*0+16] + vinserti128 m0, [lumaq+lstrideq*(1+%3)+16], 1 + mova xm1, [srcq] + vinserti128 m1, [srcq+strideq], 1 + pmaddubsw m3, m2 + pmaddubsw m0, m2 + pavgw m3, m7 + pavgw m0, m7 +%else + mova m2, [lumaq] + mova m1, [srcq] +%endif +%if %1 +%if %2 + packuswb m2, m3, m0 ; luma +%endif + punpckhbw m3, m2, m1 + punpcklbw m2, m1 ; { luma, chroma } + pmaddubsw m3, m14 + pmaddubsw m2, m14 + psraw m3, 6 + psraw m2, 6 + paddw m3, m15 + paddw m2, m15 + packuswb m2, m3 ; pack+unpack = clip +%endif +%if %1 || %2 == 0 + punpcklbw m3, m2, m7 + punpckhbw m0, m2, m7 +%endif + + ; scaling[luma_src] + pandn m4, m8, m3 + mova m6, m8 + vpgatherdd m2, [scalingq+m4-0], m8 + psrld m3, 16 + mova m8, m6 + vpgatherdd m4, [scalingq+m3-2], m6 + pandn m5, m8, m0 + mova m6, m8 + vpgatherdd m3, [scalingq+m5-0], m8 + psrld m0, 16 + mova m8, m6 + vpgatherdd m5, [scalingq+m0-2], m6 + pblendw m2, m4, 0xaa + pblendw m3, m5, 0xaa + + ; grain = grain_lut[offy+y][offx+x] +%if %2 + movu xm5, [grain_lutq+offxyq+ 0] + vinserti128 m5, [grain_lutq+offxyq+82], 1 + movd xm4, [grain_lutq+left_offxyq+ 0] + vinserti128 m4, [grain_lutq+left_offxyq+82], 1 + punpcklbw m4, m5 +%if %1 + vpbroadcastq m0, [pb_23_22] + pmaddubsw m4, m0, m4 +%else + pmaddubsw m4, m15, m4 +%endif + pmulhrsw m4, m12 + packsswb m4, m4 + vpblendd m4, m5, 0xee +%else + movu m5, [grain_lutq+offxyq] + movd xm4, [grain_lutq+left_offxyq] + punpcklbw xm4, xm5 +%if %1 + movq xm0, [pb_27_17_17_27] + pmaddubsw xm4, xm0, xm4 +%else + pmaddubsw xm4, xm15, xm4 +%endif + pmulhrsw xm4, xm12 + packsswb xm4, xm4 + vpblendd m4, m5, 0xfe +%endif + punpckhbw m5, m7 + punpcklbw m4, m7 + + ; noise = round2(scaling[luma_src] * grain, scaling_shift) + pmaddubsw m2, m4 + pmaddubsw m3, m5 + pmulhrsw m2, m9 + pmulhrsw m3, m9 + + ; unpack chroma_source + punpcklbw m0, m1, m7 + punpckhbw m1, m7 + + ; dst = clip_pixel(src, noise) + paddw m0, m2 + paddw m1, m3 + packuswb m0, m1 + pmaxub m0, m10 + pminub m0, m11 +%if %2 + mova [dstq], xm0 + vextracti128 [dstq+strideq], m0, 1 +%else + mova [dstq], m0 +%endif + +%if %2 + lea srcq, [srcq+strideq*2] + lea dstq, [dstq+strideq*2] + lea lumaq, [lumaq+lstrideq*(2<<%3)] +%else + add srcq, strideq + add dstq, strideq + add lumaq, lstrideq +%endif + add grain_lutq, 82*(1+%2) + sub hb, 1+%2 + jg %%loop_y_h_overlap + + add wq, 32>>%2 + jge .end + mov srcq, r11mp + mov dstq, r12mp + lea lumaq, [r14+wq*(1+%2)] + add srcq, wq + add dstq, wq + + ; r8m = sbym + cmp dword r8m, 0 + jne %%loop_x_hv_overlap + jmp %%loop_x_h_overlap + +%%vertical_overlap: + DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, unused, \ + sby, see, overlap, unused1, unused2, lstride + + movzx sbyd, sbyb + imul seed, [fg_dataq+FGData.seed], 0x00010001 + imul r7d, sbyd, 173 * 0x00010001 + imul sbyd, 37 * 0x01000100 + add r7d, (105 << 16) | 188 + add sbyd, (178 << 24) | (141 << 8) + and r7d, 0x00ff00ff + and sbyd, 0xff00ff00 + xor seed, r7d + xor seed, sbyd ; (cur_seed << 16) | top_seed + + DEFINE_ARGS dst, src, stride, luma, w, scaling, grain_lut, \ + unused1, unused2, see, overlap, unused3, unused4, lstride + + mov lumaq, r9mp + lea r12, [srcq+wq] + lea r13, [dstq+wq] + lea r14, [lumaq+wq*(1+%2)] + mov r11mp, r12 + mov r12mp, r13 + mov lstrideq, r10mp + neg wq + +%%loop_x_v_overlap: + ; we assume from the block above that bits 8-15 of r7d are zero'ed + mov r6d, seed + or seed, 0xeff4eff4 + test seeb, seeh + setp r7b ; parity of top_seed + shr seed, 16 + shl r7d, 16 + test seeb, seeh + setp r7b ; parity of cur_seed + or r6d, 0x00010001 + xor r7d, r6d + rorx seed, r7d, 1 ; updated (cur_seed << 16) | top_seed + + DEFINE_ARGS dst, src, stride, luma, w, scaling, grain_lut, \ + offx, offy, see, overlap, top_offxy, unused, lstride + + rorx offyd, seed, 8 + rorx offxd, seed, 12 + and offyd, 0xf000f + and offxd, 0xf000f + imul offyd, 164>>%3 + ; offxy=offy*stride+offx, (cur_offxy << 16) | top_offxy + lea offyd, [offyq+offxq*(2-%2)+0x10001*((3+(6>>%3))*82+3+(6>>%2))+(32>>%3)*82] + + DEFINE_ARGS dst, src, stride, luma, w, scaling, grain_lut, \ + h, offxy, see, overlap, top_offxy, unused, lstride + + mov grain_lutq, grain_lutmp + mov hd, hm + movzx top_offxyd, offxyw + shr offxyd, 16 +%if %2 == 0 + vpbroadcastd m13, [pb_27_17] +%endif +%%loop_y_v_overlap: + ; src +%if %2 + mova xm3, [lumaq+lstrideq*0+ 0] + vinserti128 m3, [lumaq+lstrideq*(1+%3)+ 0], 1 + vpbroadcastd m2, [pb_1] + mova xm0, [lumaq+lstrideq*0+16] + vinserti128 m0, [lumaq+lstrideq*(1+%3)+16], 1 + mova xm1, [srcq] + vinserti128 m1, [srcq+strideq], 1 + pmaddubsw m3, m2 + pmaddubsw m0, m2 + pavgw m3, m7 + pavgw m0, m7 +%else + mova m2, [lumaq] + mova m1, [srcq] +%endif +%if %1 +%if %2 + packuswb m2, m3, m0 ; luma +%endif + punpckhbw m3, m2, m1 + punpcklbw m2, m1 ; { luma, chroma } + pmaddubsw m3, m14 + pmaddubsw m2, m14 + psraw m3, 6 + psraw m2, 6 + paddw m3, m15 + paddw m2, m15 + packuswb m2, m3 ; pack+unpack = clip +%endif +%if %1 || %2 == 0 + punpcklbw m3, m2, m7 + punpckhbw m0, m2, m7 +%endif + + ; scaling[luma_src] + pandn m4, m8, m3 + mova m6, m8 + vpgatherdd m2, [scalingq+m4-0], m8 + psrld m3, 16 + mova m8, m6 + vpgatherdd m4, [scalingq+m3-2], m6 + pandn m5, m8, m0 + mova m6, m8 + vpgatherdd m3, [scalingq+m5-0], m8 + psrld m0, 16 + mova m8, m6 + vpgatherdd m5, [scalingq+m0-2], m6 + pblendw m2, m4, 0xaa + pblendw m3, m5, 0xaa + + ; grain = grain_lut[offy+y][offx+x] +%if %3 == 0 +%if %2 + movu xm0, [grain_lutq+offxyq] + vinserti128 m0, [grain_lutq+offxyq+82], 1 + movu xm4, [grain_lutq+top_offxyq] + vinserti128 m4, [grain_lutq+top_offxyq+82], 1 +%else + movu m0, [grain_lutq+offxyq] + movu m4, [grain_lutq+top_offxyq] +%endif + punpcklbw m5, m4, m0 + punpckhbw m4, m0 + pmaddubsw m5, m13, m5 + pmaddubsw m4, m13, m4 + pmulhrsw m5, m12 + pmulhrsw m4, m12 + packsswb m5, m4 +%else + movq xm4, [grain_lutq+offxyq] + vinserti128 m4, [grain_lutq+offxyq+8], 1 + movq xm5, [grain_lutq+top_offxyq] + vinserti128 m5, [grain_lutq+top_offxyq+8], 1 + punpcklbw m5, m4 + pmaddubsw m5, m13, m5 + pmulhrsw m5, m12 + vextracti128 xm4, m5, 1 + packsswb xm5, xm4 + ; only interpolate first line, insert second line unmodified + vinserti128 m5, [grain_lutq+offxyq+82], 1 +%endif + punpcklbw m4, m5, m7 + punpckhbw m5, m7 + + ; noise = round2(scaling[luma_src] * grain, scaling_shift) + pmaddubsw m2, m4 + pmaddubsw m3, m5 + pmulhrsw m2, m9 + pmulhrsw m3, m9 + + ; unpack chroma_source + punpcklbw m0, m1, m7 + punpckhbw m1, m7 + + ; dst = clip_pixel(src, noise) + paddw m0, m2 + paddw m1, m3 + packuswb m0, m1 + pmaxub m0, m10 + pminub m0, m11 +%if %2 + mova [dstq], xm0 + vextracti128 [dstq+strideq], m0, 1 +%else + mova [dstq], m0 +%endif + + sub hb, 1+%2 + jle %%end_y_v_overlap +%if %2 + lea srcq, [srcq+strideq*2] + lea dstq, [dstq+strideq*2] + lea lumaq, [lumaq+lstrideq*(2<<%3)] +%else + add srcq, strideq + add dstq, strideq + add lumaq, lstrideq +%endif + add grain_lutq, 82<<%2 +%if %2 == 0 + vpbroadcastd m13, [pb_17_27] + add hd, 0x80000000 + jnc %%loop_y_v_overlap +%endif + jmp %%loop_y + +%%end_y_v_overlap: + add wq, 32>>%2 + jge .end + mov srcq, r11mp + mov dstq, r12mp + lea lumaq, [r14+wq*(1+%2)] + add srcq, wq + add dstq, wq + + ; since fg_dataq.overlap is guaranteed to be set, we never jump + ; back to .loop_x_v_overlap, and instead always fall-through to + ; h+v overlap + +%%loop_x_hv_overlap: + ; we assume from the block above that bits 8-15 of r7d are zero'ed + mov r6d, seed + or seed, 0xeff4eff4 + test seeb, seeh + setp r7b ; parity of top_seed + shr seed, 16 + shl r7d, 16 + test seeb, seeh + setp r7b ; parity of cur_seed + or r6d, 0x00010001 + xor r7d, r6d + rorx seed, r7d, 1 ; updated (cur_seed << 16) | top_seed + + DEFINE_ARGS dst, src, stride, luma, w, scaling, grain_lut, \ + offx, offy, see, left_offxy, top_offxy, topleft_offxy, lstride + + lea topleft_offxyd, [top_offxyq+(32>>%2)] + lea left_offxyd, [offyq+(32>>%2)] + rorx offyd, seed, 8 + rorx offxd, seed, 12 + and offyd, 0xf000f + and offxd, 0xf000f + imul offyd, 164>>%3 + ; offxy=offy*stride+offx, (cur_offxy << 16) | top_offxy + lea offyd, [offyq+offxq*(2-%2)+0x10001*((3+(6>>%3))*82+3+(6>>%2))+(32>>%3)*82] + + DEFINE_ARGS dst, src, stride, luma, w, scaling, grain_lut, \ + h, offxy, see, left_offxy, top_offxy, topleft_offxy, lstride + + mov grain_lutq, grain_lutmp + mov hd, hm + movzx top_offxyd, offxyw + shr offxyd, 16 +%if %2 == 0 + vpbroadcastd m13, [pb_27_17] +%endif +%%loop_y_hv_overlap: + ; src +%if %2 + mova xm3, [lumaq+lstrideq*0+ 0] + vinserti128 m3, [lumaq+lstrideq*(1+%3)+ 0], 1 + vpbroadcastd m2, [pb_1] + mova xm0, [lumaq+lstrideq*0+16] + vinserti128 m0, [lumaq+lstrideq*(1+%3)+16], 1 + mova xm1, [srcq] + vinserti128 m1, [srcq+strideq], 1 + pmaddubsw m3, m2 + pmaddubsw m0, m2 + pavgw m3, m7 + pavgw m0, m7 +%else + mova m2, [lumaq] + mova m1, [srcq] +%endif +%if %1 +%if %2 + packuswb m2, m3, m0 ; luma +%endif + punpckhbw m3, m2, m1 + punpcklbw m2, m1 ; { luma, chroma } + pmaddubsw m3, m14 + pmaddubsw m2, m14 + psraw m3, 6 + psraw m2, 6 + paddw m3, m15 + paddw m2, m15 + packuswb m2, m3 ; pack+unpack = clip +%endif +%if %1 || %2 == 0 + punpcklbw m3, m2, m7 + punpckhbw m0, m2, m7 +%endif + + ; scaling[luma_src] + pandn m4, m8, m3 + mova m6, m8 + vpgatherdd m2, [scalingq+m4-0], m8 + psrld m3, 16 + mova m8, m6 + vpgatherdd m4, [scalingq+m3-2], m6 + pandn m5, m8, m0 + mova m6, m8 + vpgatherdd m3, [scalingq+m5-0], m8 + psrld m0, 16 + mova m8, m6 + vpgatherdd m5, [scalingq+m0-2], m6 + pblendw m2, m4, 0xaa + pblendw m3, m5, 0xaa + + ; grain = grain_lut[offy+y][offx+x] +%if %2 + movu xm4, [grain_lutq+offxyq] + vinserti128 m4, [grain_lutq+offxyq+82], 1 + movd xm0, [grain_lutq+left_offxyq] + vinserti128 m0, [grain_lutq+left_offxyq+82], 1 + movd xm6, [grain_lutq+topleft_offxyq] +%if %3 + movq xm5, [grain_lutq+top_offxyq] + vinserti128 m5, [grain_lutq+top_offxyq+8], 1 +%else + vinserti128 m6, [grain_lutq+topleft_offxyq+82], 1 + movu xm5, [grain_lutq+top_offxyq] + vinserti128 m5, [grain_lutq+top_offxyq+82], 1 +%endif + + ; do h interpolation first (so top | top/left -> top, left | cur -> cur) + punpcklbw m0, m4 +%if %3 + punpcklbw xm6, xm5 +%else + punpcklbw m6, m5 +%endif + punpcklqdq m0, m6 +%if %1 + vpbroadcastq m6, [pb_23_22] + pmaddubsw m0, m6, m0 +%else + pmaddubsw m0, m15, m0 +%endif + pmulhrsw m0, m12 + packsswb m0, m0 + vpblendd m4, m0, 0x11 +%if %3 + pshuflw xm0, xm0, q1032 + vpblendd m5, m0, 0x01 +%else + pshuflw m0, m0, q1032 + vpblendd m5, m0, 0x11 +%endif +%else + movu m4, [grain_lutq+offxyq] + movd xm0, [grain_lutq+left_offxyq] + movu m5, [grain_lutq+top_offxyq] + movd xm6, [grain_lutq+topleft_offxyq] + punpcklbw xm0, xm4 + punpcklbw xm6, xm5 + punpcklqdq xm0, xm6 +%if %1 + vpbroadcastq xm6, [pb_27_17_17_27] + pmaddubsw xm0, xm6, xm0 +%else + pmaddubsw xm0, xm15, xm0 +%endif + pmulhrsw xm0, xm12 + packsswb xm0, xm0 + vpblendd m4, m0, 0x01 + pshuflw xm0, xm0, q1032 + vpblendd m5, m0, 0x01 +%endif + + ; followed by v interpolation (top | cur -> cur) +%if %3 + vpermq m0, m4, q3120 + punpcklbw m5, m0 + pmaddubsw m5, m13, m5 + pmulhrsw m5, m12 + vextracti128 xm0, m5, 1 + packsswb xm5, xm0 + vpblendd m5, m4, 0xf0 +%else + punpckhbw m0, m5, m4 + punpcklbw m5, m4 + pmaddubsw m4, m13, m0 + pmaddubsw m5, m13, m5 + pmulhrsw m4, m12 + pmulhrsw m5, m12 + packsswb m5, m4 +%endif + punpcklbw m4, m5, m7 + punpckhbw m5, m7 + + ; noise = round2(scaling[src] * grain, scaling_shift) + pmaddubsw m2, m4 + pmaddubsw m3, m5 + pmulhrsw m2, m9 + pmulhrsw m3, m9 + + ; unpack chroma source + punpcklbw m0, m1, m7 + punpckhbw m1, m7 + + ; dst = clip_pixel(src, noise) + paddw m0, m2 + paddw m1, m3 + packuswb m0, m1 + pmaxub m0, m10 + pminub m0, m11 +%if %2 + mova [dstq], xm0 + vextracti128 [dstq+strideq], m0, 1 +%else + mova [dstq], m0 +%endif + +%if %2 + lea srcq, [srcq+strideq*2] + lea dstq, [dstq+strideq*2] + lea lumaq, [lumaq+lstrideq*(2<<%3)] +%else + add srcq, strideq + add dstq, strideq + add lumaq, lstrideq +%endif + add grain_lutq, 82<<%2 + sub hb, 1+%2 +%if %2 + jg %%loop_y_h_overlap +%else + je %%end_y_hv_overlap + vpbroadcastd m13, [pb_17_27] + add hd, 0x80000000 + jnc %%loop_y_hv_overlap + jmp %%loop_y_h_overlap +%endif + +%%end_y_hv_overlap: + add wq, 32>>%2 + jge .end + mov srcq, r11mp + mov dstq, r12mp + lea lumaq, [r14+wq*(1+%2)] + add srcq, wq + add dstq, wq + jmp %%loop_x_hv_overlap +%endmacro + + %%FGUV_32x32xN_LOOP 1, %2, %3 +.csfl: + %%FGUV_32x32xN_LOOP 0, %2, %3 +.end: + RET +%endmacro + +GEN_GRAIN_UV_FN 420, 1, 1 +FGUV_FN 420, 1, 1 +GEN_GRAIN_UV_FN 422, 1, 0 +FGUV_FN 422, 1, 0 +GEN_GRAIN_UV_FN 444, 0, 0 +FGUV_FN 444, 0, 0 + +%endif ; ARCH_X86_64 |