path: root/third_party/dav1d/src/x86/filmgrain_sse.asm

; Copyright © 2019-2021, VideoLAN and dav1d authors
; Copyright © 2019, Two Orioles, LLC
; All rights reserved.
;
; Redistribution and use in source and binary forms, with or without
; modification, are permitted provided that the following conditions are met:
;
; 1. Redistributions of source code must retain the above copyright notice, this
;    list of conditions and the following disclaimer.
;
; 2. Redistributions in binary form must reproduce the above copyright notice,
;    this list of conditions and the following disclaimer in the documentation
;    and/or other materials provided with the distribution.
;
; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
; ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
; WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
; ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
; (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
; ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
; SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

%include "config.asm"
%include "ext/x86/x86inc.asm"
%include "x86/filmgrain_common.asm"

SECTION_RODATA

pw_1024: times 8 dw 1024
pb_27_17_17_27: db 27, 17, 17, 27
                times 6 db 0, 32
pb_23_22_h: db 23, 22
            times 7 db 0, 32
pb_27_17: times 8 db 27, 17
pb_17_27: times 8 db 17, 27
pb_23_22: times 8 db 23, 22
pb_mask: db 0, 0x80, 0x80, 0, 0x80, 0, 0, 0x80, 0x80, 0, 0, 0x80, 0, 0x80, 0x80, 0
rnd_next_upperbit_mask: dw 0x100B, 0x2016, 0x402C, 0x8058
byte_blend: db 0, 0, 0, 0xff, 0, 0, 0, 0
pw_seed_xor: times 2 dw 0xb524
             times 2 dw 0x49d8
pb_1: times 4 db 1
hmul_bits: dw 32768, 16384, 8192, 4096
round: dw 2048, 1024, 512
mul_bits: dw 256, 128, 64, 32, 16
round_vals: dw 32, 64, 128, 256, 512
max: dw 255, 240, 235
min: dw 0, 16
pw_1: dw 1

%macro JMP_TABLE 2-*
    %xdefine %1_8bpc_%2_table %%table
    %xdefine %%base %1_8bpc_%2_table
    %xdefine %%prefix mangle(private_prefix %+ _%1_8bpc_%2)
    %%table:
    %rep %0 - 2
        dd %%prefix %+ .ar%3 - %%base
        %rotate 1
    %endrep
%endmacro

JMP_TABLE generate_grain_y, ssse3, 0, 1, 2, 3
JMP_TABLE generate_grain_uv_420, ssse3, 0, 1, 2, 3
JMP_TABLE generate_grain_uv_422, ssse3, 0, 1, 2, 3
JMP_TABLE generate_grain_uv_444, ssse3, 0, 1, 2, 3

SECTION .text

%if ARCH_X86_32
%define PIC_ptr(a) base+a
%else
%define PIC_ptr(a) a
%endif

%macro SCRATCH 3
%if ARCH_X86_32
    mova [rsp+%3*mmsize], m%1
%define m%2 [rsp+%3*mmsize]
%else
    SWAP             %1, %2
%endif
%endmacro

INIT_XMM ssse3
cglobal generate_grain_y_8bpc, 2, 7 + 2 * ARCH_X86_64, 16, buf, fg_data
    LEA              r4, $$
%define base r4-$$
    movq             m1, [base+rnd_next_upperbit_mask]
    movq             m4, [base+mul_bits]
    movq             m7, [base+hmul_bits]
    mov             r2d, [fg_dataq+FGData.grain_scale_shift]
    movd             m2, [base+round+r2*2]
    movd             m0, [fg_dataq+FGData.seed]
    mova             m5, [base+pb_mask]
    pshuflw          m2, m2, q0000
    pshuflw          m0, m0, q0000
    mov              r2, -73*82
    sub            bufq, r2
    lea              r3, [base+gaussian_sequence]
.loop:
    pand             m6, m0, m1
    psrlw            m3, m6, 10
    por              m6, m3            ; bits 0xf, 0x1e, 0x3c and 0x78 are set
    pmullw           m6, m4            ; bits 0x0f00 are set
    pshufb           m3, m5, m6        ; set 15th bit for next 4 seeds
    psllq            m6, m3, 30
    por              m3, m6
    psllq            m6, m3, 15
    por              m3, m6            ; aggregate each bit into next seed's high bit
    pmulhuw          m6, m0, m7
    por              m3, m6            ; 4 next output seeds
    pshuflw          m0, m3, q3333
    psrlw            m3, 5
%if ARCH_X86_64
    movq             r6, m3
    mov              r8, r6
    movzx           r5d, r6w
    shr             r6d, 16
    shr              r8, 32
    movzx            r7, r8w
    shr              r8, 16

    movd             m6, [r3+r5*2]
    pinsrw           m6, [r3+r6*2], 1
    pinsrw           m6, [r3+r7*2], 2
    pinsrw           m6, [r3+r8*2], 3
%else
    movd             r6, m3
    pshuflw          m3, m3, q3232
    movzx            r5, r6w
    shr              r6, 16

    movd             m6, [r3+r5*2]
    pinsrw           m6, [r3+r6*2], 1

    movd             r6, m3
    movzx            r5, r6w
    shr              r6, 16

    pinsrw           m6, [r3+r5*2], 2
    pinsrw           m6, [r3+r6*2], 3
%endif
    pmulhrsw         m6, m2
    packsswb         m6, m6
    movd      [bufq+r2], m6
    add              r2, 4
    jl .loop

    ; auto-regression code
    movsxd           r2, [fg_dataq+FGData.ar_coeff_lag]
    movsxd           r2, [base+generate_grain_y_8bpc_ssse3_table+r2*4]
    lea              r2, [r2+base+generate_grain_y_8bpc_ssse3_table]
    jmp              r2

.ar1:
%if ARCH_X86_32
    DEFINE_ARGS buf, fg_data, cf3, unused, val3, min, max
%elif WIN64
    DEFINE_ARGS shift, fg_data, cf3, buf, val3, min, max, x, val0
    mov            bufq, r0
%else
    DEFINE_ARGS buf, fg_data, cf3, shift, val3, min, max, x, val0
%endif
    movsx          cf3d, byte [fg_dataq+FGData.ar_coeffs_y+3]
    movd             m4, [fg_dataq+FGData.ar_coeffs_y]
    mov             ecx, [fg_dataq+FGData.ar_coeff_shift]
%if ARCH_X86_32
    mov             r1m, cf3d
    DEFINE_ARGS buf, shift, val3, min, max, x, val0
%define hd r0mp
%define cf3d r1mp
%elif WIN64
    DEFINE_ARGS shift, h, cf3, buf, val3, min, max, x, val0
%else
    DEFINE_ARGS buf, h, cf3, shift, val3, min, max, x, val0
%endif
    pxor             m6, m6
    pcmpgtb          m7, m6, m4
    punpcklbw        m4, m7
    pinsrw           m4, [base+pw_1], 3
    pshufd           m5, m4, q1111
    pshufd           m4, m4, q0000
    movd             m3, [base+round_vals+shiftq*2-12]    ; rnd
    pshuflw          m3, m3, q0000
    sub            bufq, 82*73-(82*3+79)
    mov              hd, 70
    mov            mind, -128
    mov            maxd, 127
.y_loop_ar1:
    mov              xq, -76
    movsx         val3d, byte [bufq+xq-1]
.x_loop_ar1:
    movq             m0, [bufq+xq-82-1]     ; top/left
    pcmpgtb          m7, m6, m0
    punpcklbw        m0, m7
    psrldq           m2, m0, 2              ; top
    psrldq           m1, m0, 4              ; top/right
    punpcklwd        m0, m2
    punpcklwd        m1, m3
    pmaddwd          m0, m4
    pmaddwd          m1, m5
    paddd            m0, m1
.x_loop_ar1_inner:
    movd          val0d, m0
    psrldq           m0, 4
    imul          val3d, cf3d
    add           val3d, val0d
    sar           val3d, shiftb
    movsx         val0d, byte [bufq+xq]
    add           val3d, val0d
    cmp           val3d, maxd
    cmovns        val3d, maxd
    cmp           val3d, mind
    cmovs         val3d, mind
    mov  byte [bufq+xq], val3b
    ; keep val3d in-place as left for next x iteration
    inc              xq
    jz .x_loop_ar1_end
    test             xq, 3
    jnz .x_loop_ar1_inner
    jmp .x_loop_ar1

.x_loop_ar1_end:
    add            bufq, 82
    dec              hd
    jg .y_loop_ar1
.ar0:
    RET

.ar2:
%if ARCH_X86_32
%assign stack_offset_old stack_offset
    ALLOC_STACK -16*8
%endif
    DEFINE_ARGS buf, fg_data, shift
    mov          shiftd, [fg_dataq+FGData.ar_coeff_shift]
    movd             m6, [base+round_vals-12+shiftq*2]
    movd             m7, [base+byte_blend+1]
    SCRATCH           7, 15, 7
    movq             m0, [fg_dataq+FGData.ar_coeffs_y+0]    ; cf0-7
    movd             m1, [fg_dataq+FGData.ar_coeffs_y+8]    ; cf8-11
    pxor             m7, m7
    pshuflw          m6, m6, q0000
    punpcklwd        m6, m7
    pcmpgtb          m4, m7, m0
    pcmpgtb          m5, m7, m1
    punpcklbw        m0, m4
    punpcklbw        m1, m5
    DEFINE_ARGS buf, fg_data, h, x
    pshufd           m4, m1, q0000
    pshufd           m5, m1, q1111
    pshufd           m3, m0, q3333
    pshufd           m2, m0, q2222
    pshufd           m1, m0, q1111
    pshufd           m0, m0, q0000
    SCRATCH           0, 8,  0
    SCRATCH           1, 9,  1
    SCRATCH           2, 10, 2
    SCRATCH           3, 11, 3
    SCRATCH           4, 12, 4
    SCRATCH           5, 13, 5
    SCRATCH           6, 14, 6
    sub            bufq, 82*73-(82*3+79)
    mov              hd, 70
.y_loop_ar2:
    mov              xq, -76

.x_loop_ar2:
    movq             m0, [bufq+xq-82*2-2]   ; y=-2,x=[-2,+5]
    movhps           m0, [bufq+xq-82*1-2]   ; y=-1,x=[-2,+5]
    pcmpgtb          m2, m7, m0
    punpckhbw        m1, m0, m2
    punpcklbw        m0, m2
    psrldq           m5, m0, 2              ; y=-2,x=[-1,+5]
    psrldq           m3, m1, 2              ; y=-1,x=[-1,+5]
    psrldq           m4, m1, 4              ; y=-1,x=[+0,+5]
    punpcklwd        m2, m0, m5
    punpcklwd        m3, m4
    pmaddwd          m2, m8
    pmaddwd          m3, m11
    paddd            m2, m3

    psrldq           m4, m0, 4              ; y=-2,x=[+0,+5]
    psrldq           m5, m0, 6              ; y=-2,x=[+1,+5]
    psrldq           m6, m0, 8              ; y=-2,x=[+2,+5]
    punpcklwd        m4, m5
    punpcklwd        m6, m1
    psrldq           m5, m1, 6              ; y=-1,x=[+1,+5]
    psrldq           m1, m1, 8              ; y=-1,x=[+2,+5]
    punpcklwd        m5, m1
    pmaddwd          m4, m9
    pmaddwd          m6, m10
    pmaddwd          m5, m12
    paddd            m4, m6
    paddd            m2, m5
    paddd            m2, m4
    paddd            m2, m14

    movq             m0, [bufq+xq-2]        ; y=0,x=[-2,+5]
.x_loop_ar2_inner:
    pcmpgtb          m4, m7, m0
    punpcklbw        m1, m0, m4
    pmaddwd          m3, m1, m13
    paddd            m3, m2
    psrldq           m1, 4                  ; y=0,x=0
    psrldq           m2, 4                  ; shift top to next pixel
    psrad            m3, [fg_dataq+FGData.ar_coeff_shift]
    ; don't packssdw since we only care about one value
    paddw            m3, m1
    packsswb         m3, m3
    pslldq           m3, 2
    pand             m3, m15
    pandn            m1, m15, m0
    por              m0, m1, m3
    psrldq           m0, 1
    ; overwrite 2 pixels, but that's ok
    movd      [bufq+xq-1], m0
    inc              xq
    jz .x_loop_ar2_end
    test             xq, 3
    jnz .x_loop_ar2_inner
    jmp .x_loop_ar2

.x_loop_ar2_end:
    add            bufq, 82
    dec              hd
    jg .y_loop_ar2
    RET

.ar3:
    DEFINE_ARGS buf, fg_data, shift
%if ARCH_X86_32
%assign stack_offset stack_offset_old
    ALLOC_STACK  -16*14
%elif WIN64
    SUB             rsp, 16*6
%assign stack_size_padded (stack_size_padded+16*6)
%assign stack_size (stack_size+16*6)
%else
    ALLOC_STACK  -16*6
%endif
    mov          shiftd, [fg_dataq+FGData.ar_coeff_shift]
    movd             m6, [base+round_vals-12+shiftq*2]
    movd             m7, [base+byte_blend]
    movu             m0, [fg_dataq+FGData.ar_coeffs_y+ 0]   ; cf0-15
    movq             m2, [fg_dataq+FGData.ar_coeffs_y+16]   ; cf16-23
    pxor             m3, m3
    pcmpgtb          m4, m3, m0
    pcmpgtb          m3, m2
    pshuflw          m6, m6, q0000
    SCRATCH           6, 14, 12
    SCRATCH           7, 15, 13
    punpckhbw        m1, m0, m4
    punpcklbw        m0, m4
    punpcklbw        m2, m3
    pshufd           m3, m0, q1111
    pshufd           m4, m0, q2222
    pshufd           m5, m0, q3333
    pshufd           m0, m0, q0000
    mova    [rsp+ 0*16], m0
    mova    [rsp+ 1*16], m3
    mova    [rsp+ 2*16], m4
    mova    [rsp+ 3*16], m5
    pshufd           m6, m1, q1111
    pshufd           m7, m1, q2222
    pshufd           m5, m1, q3333
    pshufd           m1, m1, q0000
    pshufd           m3, m2, q1111
    psrldq           m0, m2, 10
    pinsrw           m2, [base+pw_1], 5
    pshufd           m4, m2, q2222
    pshufd           m2, m2, q0000
    pinsrw           m0, [base+round_vals+shiftq*2-10], 3
    mova    [rsp+ 4*16], m1
    mova    [rsp+ 5*16], m6
    SCRATCH           7, 8,  6
    SCRATCH           5, 9,  7
    SCRATCH           2, 10, 8
    SCRATCH           3, 11, 9
    SCRATCH           4, 12, 10
    SCRATCH           0, 13, 11
    DEFINE_ARGS buf, fg_data, h, x
    sub            bufq, 82*73-(82*3+79)
    mov              hd, 70
.y_loop_ar3:
    mov              xq, -76

.x_loop_ar3:
    movu             m0, [bufq+xq-82*3-3]   ; y=-3,x=[-3,+12]
    pxor             m3, m3
    pcmpgtb          m3, m0
    punpckhbw        m2, m0, m3
    punpcklbw        m0, m3

    psrldq           m5, m0, 2
    psrldq           m6, m0, 4
    psrldq           m7, m0, 6
    punpcklwd        m4, m0, m5
    punpcklwd        m6, m7
    pmaddwd          m4, [rsp+ 0*16]
    pmaddwd          m6, [rsp+ 1*16]
    paddd            m4, m6

    movu             m1, [bufq+xq-82*2-3]   ; y=-2,x=[-3,+12]
    pxor             m5, m5
    pcmpgtb          m5, m1
    punpckhbw        m3, m1, m5
    punpcklbw        m1, m5
    palignr          m6, m2, m0, 10
    palignr          m7, m2, m0, 12
    psrldq           m0, 8
    punpcklwd        m0, m6
    punpcklwd        m7, m1
    pmaddwd          m0, [rsp+ 2*16]
    pmaddwd          m7, [rsp+ 3*16]
    paddd            m0, m7
    paddd            m0, m4

    psrldq           m4, m1, 2
    psrldq           m5, m1, 4
    psrldq           m6, m1, 6
    psrldq           m7, m1, 8
    punpcklwd        m4, m5
    punpcklwd        m6, m7
    pmaddwd          m4, [rsp+ 4*16]
    pmaddwd          m6, [rsp+ 5*16]
    paddd            m4, m6
    paddd            m0, m4

    movu             m2, [bufq+xq-82*1-3]   ; y=-1,x=[-3,+12]
    pxor             m7, m7
    pcmpgtb          m7, m2
    punpckhbw        m5, m2, m7
    punpcklbw        m2, m7
    palignr          m7, m3, m1, 10
    palignr          m3, m1, 12
    psrldq           m1, m2, 2
    punpcklwd        m7, m3
    punpcklwd        m3, m2, m1
    pmaddwd          m7, m8
    pmaddwd          m3, m9
    paddd            m7, m3
    paddd            m0, m7

    psrldq           m6, m2, 4
    psrldq           m1, m2, 6
    psrldq           m3, m2, 8
    palignr          m4, m5, m2, 10
    palignr          m5, m5, m2, 12

    punpcklwd        m6, m1
    punpcklwd        m3, m4
    punpcklwd        m5, m14
    pmaddwd          m6, m10
    pmaddwd          m3, m11
    pmaddwd          m5, m12
    paddd            m0, m6
    paddd            m3, m5
    paddd            m0, m3

    movq             m1, [bufq+xq-3]        ; y=0,x=[-3,+4]
.x_loop_ar3_inner:
    pxor             m5, m5
    pcmpgtb          m5, m1
    punpcklbw        m2, m1, m5
    pmaddwd          m2, m13
    pshufd           m3, m2, q1111
    paddd            m2, m3                 ; left+cur
    paddd            m2, m0                 ; add top
    psrldq           m0, 4
    psrad            m2, [fg_dataq+FGData.ar_coeff_shift]
    ; don't packssdw since we only care about one value
    packsswb         m2, m2
    pslldq           m2, 3
    pand             m2, m15
    pandn            m3, m15, m1
    por              m1, m2, m3
    movd    [bufq+xq-3], m1
    psrldq           m1, 1
    inc              xq
    jz .x_loop_ar3_end
    test             xq, 3
    jnz .x_loop_ar3_inner
    jmp .x_loop_ar3

.x_loop_ar3_end:
    add            bufq, 82
    dec              hd
    jg .y_loop_ar3
    RET

%macro generate_grain_uv_fn 3 ; ss_name, ss_x, ss_y
INIT_XMM ssse3
cglobal generate_grain_uv_%1_8bpc, 1, 7 + 3 * ARCH_X86_64, 16, buf, bufy, fg_data, uv
    movifnidn        r2, r2mp
    movifnidn        r3, r3mp
    LEA              r4, $$
%define base r4-$$
    movq             m1, [base+rnd_next_upperbit_mask]
    movq             m4, [base+mul_bits]
    movq             m7, [base+hmul_bits]
    mov             r5d, [fg_dataq+FGData.grain_scale_shift]
    movd             m6, [base+round+r5*2]
    mova             m5, [base+pb_mask]
    movd             m0, [fg_dataq+FGData.seed]
    movd             m2, [base+pw_seed_xor+uvq*4]
    pxor             m0, m2
    pshuflw          m6, m6, q0000
    pshuflw          m0, m0, q0000
    lea              r6, [base+gaussian_sequence]
%if %2
%if ARCH_X86_64
    mov             r7d, 73-35*%3
%else
    mov            r3mp, 73-35*%3
%endif
    add            bufq, 44
.loop_y:
    mov              r5, -44
.loop_x:
%else
    mov              r5, -82*73
    sub            bufq, r5
.loop:
%endif
    pand             m2, m0, m1
    psrlw            m3, m2, 10
    por              m2, m3             ; bits 0xf, 0x1e, 0x3c and 0x78 are set
    pmullw           m2, m4             ; bits 0x0f00 are set
    pshufb           m3, m5, m2         ; set 15th bit for next 4 seeds
    psllq            m2, m3, 30
    por              m3, m2
    psllq            m2, m3, 15
    por              m3, m2             ; aggregate each bit into next seed's high bit
    pmulhuw          m2, m0, m7
    por              m2, m3             ; 4 next output seeds
    pshuflw          m0, m2, q3333
    psrlw            m2, 5
%if ARCH_X86_64
    movd            r9d, m2
    pshuflw          m2, m2, q3232
    movzx            r8, r9w
    shr              r9, 16

    movd             m3, [r6+r8*2]
    pinsrw           m3, [r6+r9*2], 1

    movd            r9d, m2
    movzx            r8, r9w
    shr              r9, 16

    pinsrw           m3, [r6+r8*2], 2
    pinsrw           m3, [r6+r9*2], 3
%else
    movd             r2, m2
    pshuflw          m2, m2, q3232
    movzx            r1, r2w
    shr              r2, 16

    movd             m3, [r6+r1*2]
    pinsrw           m3, [r6+r2*2], 1

    movd             r2, m2
    movzx            r1, r2w
    shr              r2, 16

    pinsrw           m3, [r6+r1*2], 2
    pinsrw           m3, [r6+r2*2], 3
%endif
    pmulhrsw         m3, m6
    packsswb         m3, m3
    movd      [bufq+r5], m3
    add              r5, 4
%if %2
    jl .loop_x
    add            bufq, 82
%if ARCH_X86_64
    dec             r7d
%else
    dec            r3mp
%endif
    jg .loop_y
%else
    jl .loop
%endif

%if ARCH_X86_32
    mov              r2, r2mp
%endif

    ; auto-regression code
    movsxd           r5, [fg_dataq+FGData.ar_coeff_lag]
    movsxd           r5, [base+generate_grain_uv_%1_8bpc_ssse3_table+r5*4]
    lea              r5, [r5+base+generate_grain_uv_%1_8bpc_ssse3_table]
    jmp              r5

.ar0:
    DEFINE_ARGS buf, bufy, fg_data, uv, unused, shift
    movifnidn     bufyq, bufymp
%if ARCH_X86_32
%assign stack_offset_old stack_offset
    ALLOC_STACK   -2*16
%endif
    imul            uvd, 28
    mov          shiftd, [fg_dataq+FGData.ar_coeff_shift]
    movd             m5, [fg_dataq+FGData.ar_coeffs_uv+uvq]
    movd             m4, [base+hmul_bits+shiftq*2]
    DEFINE_ARGS buf, bufy, h, x
    pxor             m0, m0
    pcmpgtb          m0, m5
    punpcklbw        m5, m0
    movd             m7, [base+pb_1]
%if %2
    movd             m6, [base+hmul_bits+2+%3*2]
%endif
    pshuflw          m5, m5, q0000
    pshuflw          m4, m4, q0000
    pshufd           m7, m7, q0000
%if %2
    pshuflw          m6, m6, q0000
%endif
    punpcklqdq       m5, m5
    punpcklqdq       m4, m4
%if %2
    punpcklqdq       m6, m6
%endif
    pcmpeqw          m1, m1
    pslldq           m1, 12>>%2
    SCRATCH           1, 8, 0
    SCRATCH           4, 9, 1
%if %2
    sub            bufq, 82*(73-35*%3)+82-(82*3+41)
%else
    sub            bufq, 82*70-3
%endif
    add           bufyq, 3+82*3
    mov              hd, 70-35*%3
.y_loop_ar0:
    xor              xd, xd
.x_loop_ar0:
    ; first 32 pixels
%if %2
    movu             m1, [bufyq+xq*2]
%if %3
    movu             m2, [bufyq+xq*2+82]
%endif
    movu             m3, [bufyq+xq*2+16]
%if %3
    movu             m4, [bufyq+xq*2+82+16]
%endif
    pmaddubsw        m0, m7, m1
%if %3
    pmaddubsw        m1, m7, m2
%endif
    pmaddubsw        m2, m7, m3
%if %3
    pmaddubsw        m3, m7, m4
    paddw            m0, m1
    paddw            m2, m3
%endif
    pmulhrsw         m0, m6
    pmulhrsw         m2, m6
%else
    movu             m0, [bufyq+xq]
    pxor             m6, m6
    pcmpgtb          m6, m0
    punpckhbw        m2, m0, m6
    punpcklbw        m0, m6
%endif
    pmullw           m0, m5
    pmullw           m2, m5
    pmulhrsw         m0, m9
    pmulhrsw         m2, m9
    movu             m1, [bufq+xq]
    pxor             m4, m4
    pcmpgtb          m4, m1
    punpckhbw        m3, m1, m4
%if %2
    punpcklbw        m1, m4
    paddw            m2, m3
    paddw            m0, m1
%else
    punpcklbw        m6, m1, m4
    paddw            m2, m3
    paddw            m0, m6
%endif
    packsswb         m0, m2
%if %2
    movu      [bufq+xq], m0
    add              xd, 16
    cmp              xd, 32
    jl .x_loop_ar0

    ; last 6/12 pixels
    movu             m1, [bufyq+xq*(1+%2)]
%if %3
    movu             m2, [bufyq+xq*2+82]
%endif
    pmaddubsw        m0, m7, m1
%if %3
    pmaddubsw        m1, m7, m2
    paddw            m0, m1
%endif
    pmulhrsw         m0, m6
    pmullw           m0, m5
    pmulhrsw         m0, m9
    movq             m1, [bufq+xq]
    pxor             m4, m4
    pcmpgtb          m4, m1
    punpcklbw        m2, m1, m4
    paddw            m0, m2
    packsswb         m0, m0
    pandn            m2, m8, m0
    pand             m1, m8
    por              m2, m1
    movq      [bufq+xq], m2
%else
    add              xd, 16
    cmp              xd, 80
    je .y_loop_final_ar0
    movu   [bufq+xq-16], m0
    jmp .x_loop_ar0
.y_loop_final_ar0:
    pandn            m2, m8, m0
    pand             m1, m8
    por              m2, m1
    movu   [bufq+xq-16], m2
%endif

    add            bufq, 82
    add           bufyq, 82<<%3
    dec              hd
    jg .y_loop_ar0
    RET

.ar1:
%if ARCH_X86_32
%assign stack_offset stack_offset_old
%assign stack_size_padded 0
%xdefine rstk rsp
%endif
    DEFINE_ARGS buf, bufy, fg_data, uv, val3, cf3, min, max, x
    imul            uvd, 28
    movsx          cf3d, byte [fg_dataq+FGData.ar_coeffs_uv+uvq+3]
    movd             m4, [fg_dataq+FGData.ar_coeffs_uv+uvq-1]
    pinsrw           m4, [fg_dataq+FGData.ar_coeffs_uv+uvq+4], 2
%if ARCH_X86_32
    mov            r3mp, cf3d
    DEFINE_ARGS buf, shift, fg_data, val3, min, max, x
%elif WIN64
    DEFINE_ARGS shift, bufy, fg_data, buf, val3, cf3, min, max, x
    mov            bufq, r0
%else
    DEFINE_ARGS buf, bufy, fg_data, shift, val3, cf3, min, max, x
%endif
    mov          shiftd, [fg_dataq+FGData.ar_coeff_shift]
    movd             m3, [base+round_vals+shiftq*2-12]    ; rnd
%if %2
    movd             m7, [base+pb_1]
    movd             m6, [base+hmul_bits+2+%3*2]
%endif
    psrldq           m4, 1
%if ARCH_X86_32
    DEFINE_ARGS buf, shift, val0, val3, min, max, x
%elif WIN64
    DEFINE_ARGS shift, bufy, h, buf, val3, cf3, min, max, x, val0
%else
    DEFINE_ARGS buf, bufy, h, shift, val3, cf3, min, max, x, val0
%endif
    pxor             m5, m5
    punpcklwd        m3, m5
%if %2
    punpcklwd        m6, m6
%endif
    pcmpgtb          m5, m4
    punpcklbw        m4, m5
    pshufd           m5, m4, q1111
    pshufd           m4, m4, q0000
    pshufd           m3, m3, q0000
%if %2
    pshufd           m7, m7, q0000
    pshufd           m6, m6, q0000
    sub            bufq, 82*(73-35*%3)+44-(82*3+41)
%else
    sub            bufq, 82*69+3
%endif
%if ARCH_X86_32
    add            r1mp, 79+82*3
    mov            r0mp, 70-35*%3
%else
    add           bufyq, 79+82*3
    mov              hd, 70-35*%3
%endif
    mov            mind, -128
    mov            maxd, 127
.y_loop_ar1:
    mov              xq, -(76>>%2)
    movsx         val3d, byte [bufq+xq-1]
.x_loop_ar1:
%if %2
%if ARCH_X86_32
    mov              r2, r1mp
    movq             m0, [r2+xq*2]
%if %3
    movq             m1, [r2+xq*2+82]
%endif
%else
    movq             m0, [bufyq+xq*2]
%if %3
    movq             m1, [bufyq+xq*2+82]
%endif
%endif
    pmaddubsw        m2, m7, m0
%if %3
    pmaddubsw        m0, m7, m1
    paddw            m2, m0
%endif
    pmulhrsw         m2, m6
%else
%if ARCH_X86_32
    mov              r2, r1mp
    movd             m2, [r2+xq]
%else
    movd             m2, [bufyq+xq]
%endif
    pxor             m0, m0
    pcmpgtb          m0, m2
    punpcklbw        m2, m0
%endif

    movq             m0, [bufq+xq-82-1]     ; top/left
    pxor             m1, m1
    pcmpgtb          m1, m0
    punpcklbw        m0, m1
    psrldq           m1, m0, 4              ; top/right
    punpcklwd        m1, m2
    psrldq           m2, m0, 2              ; top
    punpcklwd        m0, m2
    pmaddwd          m0, m4
    pmaddwd          m1, m5
    paddd            m0, m1
    paddd            m0, m3
.x_loop_ar1_inner:
    movd          val0d, m0
    psrldq           m0, 4
%if ARCH_X86_32
    imul          val3d, r3mp
%else
    imul          val3d, cf3d
%endif
    add           val3d, val0d
    sar           val3d, shiftb
    movsx         val0d, byte [bufq+xq]
    add           val3d, val0d
    cmp           val3d, maxd
    cmovns        val3d, maxd
    cmp           val3d, mind
    cmovs         val3d, mind
    mov  byte [bufq+xq], val3b
    ; keep val3d in-place as left for next x iteration
    inc              xq
    jz .x_loop_ar1_end
    test             xq, 3
    jnz .x_loop_ar1_inner
    jmp .x_loop_ar1

.x_loop_ar1_end:
    add            bufq, 82
%if ARCH_X86_32
    add            r1mp, 82<<%3
    dec            r0mp
%else
    add           bufyq, 82<<%3
    dec              hd
%endif
    jg .y_loop_ar1
    RET

.ar2:
%if ARCH_X86_32
%assign stack_offset stack_offset_old
%assign stack_size_padded 0
%xdefine rstk rsp
    ALLOC_STACK   -8*16
%endif
    DEFINE_ARGS buf, bufy, fg_data, uv, unused, shift
    movifnidn     bufyq, bufymp
    mov          shiftd, [fg_dataq+FGData.ar_coeff_shift]
    imul            uvd, 28
    movd             m7, [base+round_vals-12+shiftq*2]
    movu             m0, [fg_dataq+FGData.ar_coeffs_uv+uvq+0]   ; cf0-12
    pxor             m2, m2
    pcmpgtb          m2, m0
    punpckhbw        m1, m0, m2
    punpcklbw        m0, m2
    pinsrw           m1, [base+pw_1], 5
    punpcklwd        m7, m7
    pshufd           m7, m7, q0000
    DEFINE_ARGS buf, bufy, fg_data, h, unused, x
    pshufd           m4, m1, q0000
    pshufd           m5, m1, q1111
    pshufd           m6, m1, q2222
    pshufd           m3, m0, q3333
    pshufd           m2, m0, q2222
    pshufd           m1, m0, q1111
    pshufd           m0, m0, q0000
    SCRATCH           0, 8,  0
    SCRATCH           1, 9,  1
    SCRATCH           2, 10, 2
    SCRATCH           3, 11, 3
    SCRATCH           4, 12, 4
    SCRATCH           5, 13, 5
    SCRATCH           6, 14, 6
    SCRATCH           7, 15, 7
%if %2
    movd             m7, [base+hmul_bits+2+%3*2]
    movd             m6, [base+pb_1]
    punpcklwd        m7, m7
    pshufd           m6, m6, q0000
    pshufd           m7, m7, q0000
    sub            bufq, 82*(73-35*%3)+44-(82*3+41)
%else
    sub            bufq, 82*69+3
%endif
    add           bufyq, 79+82*3
    mov              hd, 70-35*%3
.y_loop_ar2:
    mov              xq, -(76>>%2)

.x_loop_ar2:
    pxor             m2, m2
    movq             m0, [bufq+xq-82*2-2]   ; y=-2,x=[-2,+5]
    movhps           m0, [bufq+xq-82*1-2]   ; y=-1,x=[-2,+5]
    pcmpgtb          m2, m0
    punpckhbw        m1, m0, m2
    punpcklbw        m0, m2
    psrldq           m5, m0, 2              ; y=-2,x=[-1,+5]
    psrldq           m3, m1, 2              ; y=-1,x=[-1,+5]
    psrldq           m4, m1, 4              ; y=-1,x=[+0,+5]
    punpcklwd        m2, m0, m5
    punpcklwd        m3, m4
    pmaddwd          m2, m8
    pmaddwd          m3, m11
    paddd            m2, m3

    psrldq           m4, m0, 4              ; y=-2,x=[+0,+5]
    psrldq           m5, m0, 6              ; y=-2,x=[+1,+5]
    psrldq           m0, 8                  ; y=-2,x=[+2,+5]
    punpcklwd        m4, m5
    punpcklwd        m0, m1
    psrldq           m3, m1, 6              ; y=-1,x=[+1,+5]
    psrldq           m1, m1, 8              ; y=-1,x=[+2,+5]
    punpcklwd        m3, m1
    pmaddwd          m4, m9
    pmaddwd          m0, m10
    pmaddwd          m3, m12
    paddd            m4, m0
    paddd            m2, m3
    paddd            m2, m4

%if %2
    movq             m1, [bufyq+xq*2]
%if %3
    movq             m3, [bufyq+xq*2+82]
%endif
    pmaddubsw        m0, m6, m1
%if %3
    pmaddubsw        m1, m6, m3
    paddw            m0, m1
%endif
    pmulhrsw         m0, m7
%else
    movd             m0, [bufyq+xq]
    pxor             m1, m1
    pcmpgtb          m1, m0
    punpcklbw        m0, m1
%endif
    punpcklwd        m0, m15
    pmaddwd          m0, m14
    paddd            m2, m0

    movq             m0, [bufq+xq-2]        ; y=0,x=[-2,+5]
    pxor             m4, m4
    movd             m5, [base+byte_blend+1]
    punpcklbw        m5, m5
.x_loop_ar2_inner:
    pcmpgtb          m1, m4, m0
    punpcklbw        m0, m1
    pmaddwd          m3, m0, m13
    paddd            m3, m2
    psrldq           m2, 4                  ; shift top to next pixel
    psrad            m3, [fg_dataq+FGData.ar_coeff_shift]
    pslldq           m3, 4
    pand             m3, m5
    paddw            m0, m3
    packsswb         m0, m0
    movd    [bufq+xq-2], m0
    psrldq           m0, 1
    inc              xq
    jz .x_loop_ar2_end
    test             xq, 3
    jnz .x_loop_ar2_inner
    jmp .x_loop_ar2

.x_loop_ar2_end:
    add            bufq, 82
    add           bufyq, 82<<%3
    dec              hd
    jg .y_loop_ar2
    RET

.ar3:
%if ARCH_X86_32
%assign stack_offset stack_offset_old
%assign stack_size_padded 0
%xdefine rstk rsp
%endif
    DEFINE_ARGS buf, bufy, fg_data, uv, unused, shift
    movifnidn     bufyq, bufymp
%if ARCH_X86_32
    ALLOC_STACK  -15*16
%else
    SUB             rsp, 16*7
%assign stack_size_padded (stack_size_padded+16*7)
%assign stack_size (stack_size+16*7)
%endif
    mov          shiftd, [fg_dataq+FGData.ar_coeff_shift]
    imul            uvd, 28

    movu             m0, [fg_dataq+FGData.ar_coeffs_uv+uvq+ 0]   ; cf0-15
    pxor             m3, m3
    pcmpgtb          m3, m0
    punpckhbw        m1, m0, m3
    punpcklbw        m0, m3
    pshufd           m2, m0, q1111
    pshufd           m3, m0, q2222
    pshufd           m4, m0, q3333
    pshufd           m0, m0, q0000
    pshufd           m5, m1, q1111
    pshufd           m6, m1, q2222
    pshufd           m7, m1, q3333
    pshufd           m1, m1, q0000
    mova    [rsp+ 0*16], m0
    mova    [rsp+ 1*16], m2
    mova    [rsp+ 2*16], m3
    mova    [rsp+ 3*16], m4
    mova    [rsp+ 4*16], m1
    mova    [rsp+ 5*16], m5
    mova    [rsp+ 6*16], m6
    SCRATCH           7, 8, 7

    movu             m2, [fg_dataq+FGData.ar_coeffs_uv+uvq+16]   ; cf16-24 [24=luma]
    pxor             m4, m4
    pcmpgtb          m4, m2
    punpckhbw        m5, m2, m4
    punpcklbw        m2, m4
    pshufd           m4, m2, q3232
    punpcklwd        m3, m4, m5
    pshuflw          m5, m4, q3321
    pshufd           m4, m3, q0000
    pshufd           m3, m2, q1111
    pshufd           m2, m2, q0000
    pinsrw           m5, [base+round_vals+shiftq*2-10], 3
    SCRATCH           2, 9,  8
    SCRATCH           3, 10, 9
    SCRATCH           4, 11, 10
    SCRATCH           5, 12, 11

    movd             m2, [base+round_vals-12+shiftq*2]
%if %2
    movd             m1, [base+pb_1]
    movd             m3, [base+hmul_bits+2+%3*2]
%endif
    pxor             m0, m0
    punpcklwd        m2, m0
%if %2
    punpcklwd        m3, m3
%endif
    pshufd           m2, m2, q0000
%if %2
    pshufd           m1, m1, q0000
    pshufd           m3, m3, q0000
    SCRATCH           1, 13, 12
%endif
    SCRATCH           2, 14, 13
%if %2
    SCRATCH           3, 15, 14
%endif

    DEFINE_ARGS buf, bufy, fg_data, h, unused, x
%if %2
    sub            bufq, 82*(73-35*%3)+44-(82*3+41)
%else
    sub            bufq, 82*69+3
%endif
    add           bufyq, 79+82*3
    mov              hd, 70-35*%3
.y_loop_ar3:
    mov              xq, -(76>>%2)

.x_loop_ar3:
    movu             m0, [bufq+xq-82*3-3]   ; y=-3,x=[-3,+12]
    pxor             m4, m4
    pcmpgtb          m4, m0
    punpckhbw        m3, m0, m4
    punpcklbw        m0, m4

    psrldq           m5, m0, 2
    psrldq           m6, m0, 4
    psrldq           m7, m0, 6
    punpcklwd        m4, m0, m5
    punpcklwd        m6, m7
    pmaddwd          m4, [rsp+ 0*16]
    pmaddwd          m6, [rsp+ 1*16]
    paddd            m4, m6

    palignr          m2, m3, m0, 10
    palignr          m3, m0, 12
    psrldq           m0, 8

    movu             m1, [bufq+xq-82*2-3]   ; y=-2,x=[-3,+12]
    pxor             m6, m6
    pcmpgtb          m6, m1
    punpckhbw        m5, m1, m6
    punpcklbw        m1, m6

    punpcklwd        m0, m2
    punpcklwd        m3, m1
    pmaddwd          m0, [rsp+ 2*16]
    pmaddwd          m3, [rsp+ 3*16]
    paddd            m0, m3
    paddd            m0, m4

    movu             m2, [bufq+xq-82*1-3]   ; y=-1,x=[-3,+12]
    pxor             m7, m7
    pcmpgtb          m7, m2
    punpckhbw        m6, m2, m7
    punpcklbw        m2, m7

    palignr          m3, m5, m1, 10
    palignr          m5, m1, 12
    psrldq           m4, m2, 2

    punpcklwd        m3, m5
    punpcklwd        m5, m2, m4
    pmaddwd          m3, [rsp+ 6*16]
    pmaddwd          m5, m8
    paddd            m3, m5
    paddd            m0, m3

    psrldq           m3, m1, 2
    psrldq           m4, m1, 4
    psrldq           m5, m1, 6
    psrldq           m1, 8

    punpcklwd        m3, m4
    punpcklwd        m5, m1
    pmaddwd          m3, [rsp+ 4*16]
    pmaddwd          m5, [rsp+ 5*16]
    paddd            m3, m5
    paddd            m0, m3

%if %2
    movq             m1, [bufyq+xq*2]
%if %3
    movq             m3, [bufyq+xq*2+82]
%endif
    pmaddubsw        m7, m13, m1
%if %3
    pmaddubsw        m5, m13, m3
    paddw            m7, m5
%endif
    pmulhrsw         m7, m15
%else
    movd             m7, [bufyq+xq]
    pxor             m1, m1
    pcmpgtb          m1, m7
    punpcklbw        m7, m1
%endif

    psrldq           m1, m2, 4
    psrldq           m3, m2, 6
    palignr          m4, m6, m2, 10
    palignr          m6, m2, 12
    psrldq           m2, 8

    punpcklwd        m1, m3
    punpcklwd        m2, m4
    punpcklwd        m6, m7
    pmaddwd          m1, m9
    pmaddwd          m2, m10
    pmaddwd          m6, m11
    paddd            m1, m2
    paddd            m0, m6
    paddd            m0, m1
    paddd            m0, m14

    movq             m1, [bufq+xq-3]        ; y=0,x=[-3,+4]
    pxor             m4, m4
    movd             m5, [base+byte_blend]
.x_loop_ar3_inner:
    pcmpgtb          m2, m4, m1
    punpcklbw        m3, m1, m2
    pmaddwd          m2, m3, m12
    pshufd           m3, m2, q1111
    paddd            m2, m3                 ; left+cur
    paddd            m2, m0                 ; add top
    psrldq           m0, 4
    psrad            m2, [fg_dataq+FGData.ar_coeff_shift]
    ; don't packssdw, we only care about one value
    packsswb         m2, m2
    pandn            m3, m5, m1
    pslld            m2, 24
    pand             m2, m5
    por              m1, m2, m3
    movd    [bufq+xq-3], m1
    psrldq           m1, 1
    inc              xq
    jz .x_loop_ar3_end
    test             xq, 3
    jnz .x_loop_ar3_inner
    jmp .x_loop_ar3

.x_loop_ar3_end:
    add            bufq, 82
    add           bufyq, 82<<%3
    dec              hd
    jg .y_loop_ar3
    RET
%endmacro

generate_grain_uv_fn 420, 1, 1
generate_grain_uv_fn 422, 1, 0
generate_grain_uv_fn 444, 0, 0

%macro vpgatherdw 5-6 ; dst, src, base, tmp_gpr[x2], tmp_xmm_reg
%assign %%idx 0
%define %%tmp %2
%if %0 == 6
%define %%tmp %6
%endif
%rep 4
%if %%idx == 0
    movd        %5 %+ d, %2
    pshuflw       %%tmp, %2, q3232
%else
    movd        %5 %+ d, %%tmp
%if %%idx == 2
    punpckhqdq    %%tmp, %%tmp
%elif %%idx == 4
    psrlq         %%tmp, 32
%endif
%endif
    movzx       %4 %+ d, %5 %+ w
    shr         %5 %+ d, 16

%if %%idx == 0
    movd             %1, [%3+%4]
%else
    pinsrw           %1, [%3+%4], %%idx + 0
%endif
    pinsrw           %1, [%3+%5], %%idx + 1
%assign %%idx %%idx+2
%endrep
%endmacro

INIT_XMM ssse3
; fgy_32x32xn(dst, src, stride, fg_data, w, scaling, grain_lut, h, sby)
%if ARCH_X86_32
%if STACK_ALIGNMENT < mmsize
cglobal fgy_32x32xn_8bpc, 0, 7, 16, 0 - (5 * mmsize + 16 * gprsize), \
        dst, src, scaling, unused1, fg_data, picptr, unused2
    ; copy stack arguments to new position post-alignment, so that we
    ; don't have to keep the old stack location in a separate register
    mov              r0, r0m
    mov              r1, r2m
    mov              r2, r4m
    mov              r3, r6m
    mov              r4, r7m
    mov              r5, r8m

    mov [rsp+5*mmsize+ 4*gprsize], r0
    mov [rsp+5*mmsize+ 6*gprsize], r1
    mov [rsp+5*mmsize+ 8*gprsize], r2
    mov [rsp+5*mmsize+10*gprsize], r3
    mov [rsp+5*mmsize+11*gprsize], r4
    mov [rsp+5*mmsize+12*gprsize], r5
%else
cglobal fgy_32x32xn_8bpc, 0, 7, 16, 5 * mmsize + 4 * gprsize, \
        dst, src, scaling, unused1, fg_data, picptr, unused2
%endif
    mov            srcq, srcm
    mov        fg_dataq, r3m
    mov        scalingq, r5m
%if STACK_ALIGNMENT < mmsize
%define r0m [rsp+5*mmsize+ 4*gprsize]
%define r1m [rsp+5*mmsize+ 5*gprsize]
%define r2m [rsp+5*mmsize+ 6*gprsize]
%define r3m [rsp+5*mmsize+ 7*gprsize]
%define r4m [rsp+5*mmsize+ 8*gprsize]
%define r5m [rsp+5*mmsize+ 9*gprsize]
%define r6m [rsp+5*mmsize+10*gprsize]
%define r7m [rsp+5*mmsize+11*gprsize]
%define r8m [rsp+5*mmsize+12*gprsize]
%endif
    LEA              r5, pb_mask
%define base r5-pb_mask
    mov             r5m, picptrq
%else
cglobal fgy_32x32xn_8bpc, 6, 15, 16, dst, src, stride, fg_data, w, scaling, grain_lut
    lea              r7, [pb_mask]
%define base r7-pb_mask
%endif
    mov             r6d, [fg_dataq+FGData.scaling_shift]
    movd             m3, [base+mul_bits+r6*2-14]
    mov             r6d, [fg_dataq+FGData.clip_to_restricted_range]
    movd             m4, [base+max+r6*4]
    movd             m5, [base+min+r6*2]
    punpcklwd        m3, m3
    punpcklwd        m4, m4
    punpcklwd        m5, m5
    pshufd           m3, m3, q0000
    pshufd           m4, m4, q0000
    pshufd           m5, m5, q0000
    SCRATCH           3, 11, 0
    SCRATCH           4, 12, 1
    SCRATCH           5, 13, 2

%if ARCH_X86_32
    DEFINE_ARGS dst, src, scaling, sby, fg_data, picptr, overlap
%else
    DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, unused, sby, see, overlap
%endif

    mov            sbyd, r8m
    mov        overlapd, [fg_dataq+FGData.overlap_flag] ; left_overlap: overlap & 1
    test       overlapd, overlapd
    jz .no_vertical_overlap
    mova             m6, [base+pw_1024]
    mova             m7, [base+pb_27_17_17_27]
    SCRATCH           6, 14, 3
    SCRATCH           7, 15, 4
    test           sbyd, sbyd
    jnz .vertical_overlap
    ; fall-through

.no_vertical_overlap:
    mov             r8m, overlapd
%if ARCH_X86_32
    DEFINE_ARGS dst, src, scaling, see, fg_data, picptr, unused
    imul           seed, (173 << 24) | 37
%else
    imul           seed, sbyd, (173 << 24) | 37
%endif
    add            seed, (105 << 24) | 178
    rol            seed, 8
    movzx          seed, seew
    xor            seed, [fg_dataq+FGData.seed]

%if ARCH_X86_32
    DEFINE_ARGS dst, src, scaling, see, w, picptr, src_bak

    mov             r3m, seed
    mov              wq, r4m
%else
    DEFINE_ARGS dst, src, stride, src_bak, w, scaling, grain_lut, \
                unused1, unused2, see, unused3
%endif

    lea        src_bakq, [srcq+wq]
    neg              wq
    sub           dstmp, srcq
%if ARCH_X86_32
    mov             r1m, src_bakq
    mov             r4m, wq
    DEFINE_ARGS dst, src, scaling, see, unused1, unused2, unused3
%endif

.loop_x:
%if ARCH_X86_32
    mov            seed, r3m
%endif
    mov             r6d, seed
    or             seed, 0xEFF4
    shr             r6d, 1
    test           seeb, seeh
    lea            seed, [r6+0x8000]
    cmovp          seed, r6d                ; updated seed
%if ARCH_X86_32
    mov             r3m, seed

    DEFINE_ARGS dst, src, scaling, offy, unused1, unused2, offx

    mov           offxd, offyd
%else
    DEFINE_ARGS dst, src, stride, src_bak, w, scaling, grain_lut, \
                offx, offy, see, unused

    mov           offyd, seed
    mov           offxd, seed
%endif
    ror           offyd, 8
    shr           offxd, 12
    and           offyd, 0xf
    imul          offyd, 164
    lea           offyq, [offyq+offxq*2+747] ; offy*stride+offx

%if ARCH_X86_32
    ; r0m=dst, r1m=src_bak, r2m=stride, r3m=see, r4m=w, r5m=picptr,
    ; r6m=grain_lut, r7m=h, r8m=overlap_v|h
    DEFINE_ARGS dst, src, scaling, offxy, h, picptr, grain_lut
%else
    DEFINE_ARGS dst, src, stride, src_bak, w, scaling, grain_lut, \
                h, offxy, see, unused
%endif

.loop_x_odd:
    mov              hd, r7m
    mov      grain_lutq, grain_lutmp
.loop_y:
    ; src
    mova             m0, [srcq]
    pxor             m2, m2
    punpckhbw        m1, m0, m2
    punpcklbw        m0, m2                 ; m0-1: src as word

    ; scaling[src]
%if ARCH_X86_32
    vpgatherdw       m4, m0, scalingq-1, r0, r5, m3
    vpgatherdw       m5, m1, scalingq-1, r0, r5, m3
%else
    vpgatherdw       m4, m0, scalingq-1, r12, r13, m3
    vpgatherdw       m5, m1, scalingq-1, r12, r13, m3
%endif
    REPX {psrlw x, 8}, m4, m5

    ; grain = grain_lut[offy+y][offx+x]
    movu             m3, [grain_lutq+offxyq]
    pcmpgtb          m7, m2, m3
    punpcklbw        m2, m3, m7
    punpckhbw        m3, m7

    ; noise = round2(scaling[src] * grain, scaling_shift)
    pmullw           m2, m4
    pmullw           m3, m5
    pmulhrsw         m2, m11
    pmulhrsw         m3, m11

    ; dst = clip_pixel(src, noise)
    paddw            m0, m2
    paddw            m1, m3
    pmaxsw           m0, m13
    pmaxsw           m1, m13
    pminsw           m0, m12
    pminsw           m1, m12
    packuswb         m0, m1
    movifnidn      dstq, dstmp
    mova    [dstq+srcq], m0

    add            srcq, r2mp
    add      grain_lutq, 82
    dec              hd
    jg .loop_y

%if ARCH_X86_32
    add            r4mp, 16
%else
    add              wq, 16
%endif
    jge .end
%if ARCH_X86_32
    mov            srcq, r1mp
    add            srcq, r4mp
%else
    lea            srcq, [src_bakq+wq]
%endif
    btc       dword r8m, 2
    jc .next_blk

    add          offxyd, 16
    test      dword r8m, 2              ; r8m & 2 = have_top_overlap
    jz .loop_x_odd

%if ARCH_X86_32
    add dword [rsp+5*mmsize+1*gprsize], 16
%else
    add            r11d, 16             ; top_offxyd
%endif
    jnz .loop_x_odd_v_overlap

.next_blk:
    test      dword r8m, 1
    jz .loop_x

    test      dword r8m, 2
    jnz .loop_x_hv_overlap

    ; horizontal overlap (without vertical overlap)
.loop_x_h_overlap:
%if ARCH_X86_32
    ; r0m=dst, r1m=src_bak, r2m=stride, r3m=see, r4m=w, r5m=picptr,
    ; r6m=grain_lut, r7m=h, r8m=overlap_v|h
    DEFINE_ARGS dst, src, scaling, offxy, unused1, unused2, unused3

    add          offxyd, 16                 ; left_offxyd
    mov [rsp+5*mmsize+0*gprsize], offxyd

    DEFINE_ARGS dst, src, scaling, see, unused1, unused2, unused3

    mov            seed, r3m
%else
    DEFINE_ARGS dst, src, stride, src_bak, w, scaling, grain_lut, \
                offx, offy, see, left_offxy

    lea     left_offxyd, [offyd+16]         ; previous column's offy*stride+offx
%endif

    mov             r6d, seed
    or             seed, 0xEFF4
    shr             r6d, 1
    test           seeb, seeh
    lea            seed, [r6+0x8000]
    cmovp          seed, r6d                ; updated seed

%if ARCH_X86_32
    mov             r3m, seed

    DEFINE_ARGS dst, src, scaling, offy, unused1, unused2, offx

    mov           offxd, offyd
%else
    mov           offyd, seed
    mov           offxd, seed
%endif
    ror           offyd, 8
    shr           offxd, 12
    and           offyd, 0xf
    imul          offyd, 164
    lea           offyq, [offyq+offxq*2+747] ; offy*stride+offx

%if ARCH_X86_32
    DEFINE_ARGS dst, src, scaling, offxy, h, picptr, grain_lut
%else
    DEFINE_ARGS dst, src, stride, src_bak, w, scaling, grain_lut, \
                h, offxy, see, left_offxy
%endif

    mov              hd, r7m
    mov      grain_lutq, grain_lutmp
.loop_y_h_overlap:
    ; src
    mova             m0, [srcq]
    pxor             m2, m2
    punpckhbw        m1, m0, m2
    punpcklbw        m0, m2                 ; m0-1: src as word

    ; scaling[src]
%if ARCH_X86_32
    vpgatherdw       m4, m0, scalingq-1, r0, r5, m3
    vpgatherdw       m5, m1, scalingq-1, r0, r5, m3
%else
    vpgatherdw       m4, m0, scalingq-1, r12, r13, m3
    vpgatherdw       m5, m1, scalingq-1, r12, r13, m3
%endif
    REPX {psrlw x, 8}, m4, m5

    ; grain = grain_lut[offy+y][offx+x]
    movu             m3, [grain_lutq+offxyq]
%if ARCH_X86_32
    mov              r5, [rsp+5*mmsize+0*gprsize]
    movd             m7, [grain_lutq+r5]
%else
    movd             m7, [grain_lutq+left_offxyq]
%endif
    punpcklbw        m7, m3
    pmaddubsw        m6, m15, m7
    pmulhrsw         m6, m14
    packsswb         m6, m6
    shufps           m6, m3, q3210
    pcmpgtb          m2, m6
    punpcklbw        m7, m6, m2
    punpckhbw        m6, m2

    ; noise = round2(scaling[src] * grain, scaling_shift)
    pmullw           m7, m4
    pmullw           m6, m5
    pmulhrsw         m7, m11
    pmulhrsw         m6, m11

    ; dst = clip_pixel(src, noise)
    paddw            m0, m7
    paddw            m1, m6
    pmaxsw           m0, m13
    pmaxsw           m1, m13
    pminsw           m0, m12
    pminsw           m1, m12
    packuswb         m0, m1
    movifnidn      dstq, dstmp
    mova    [dstq+srcq], m0

    add            srcq, r2mp
    add      grain_lutq, 82
    dec              hd
    jg .loop_y_h_overlap

%if ARCH_X86_32
    add            r4mp, 16
%else
    add              wq, 16
%endif
    jge .end
%if ARCH_X86_32
    mov            srcq, r1m
    add            srcq, r4m
%else
    lea            srcq, [src_bakq+wq]
%endif
    xor       dword r8m, 4
    add          offxyd, 16

    ; since this half-block had left-overlap, the next does not
    test      dword r8m, 2              ; have_top_overlap
    jz .loop_x_odd
%if ARCH_X86_32
    add dword [rsp+5*mmsize+1*gprsize], 16
%else
    add            r11d, 16             ; top_offxyd
%endif
    jmp .loop_x_odd_v_overlap

.end:
    RET

.vertical_overlap:
%if ARCH_X86_32
    DEFINE_ARGS dst, src, scaling, sby, fg_data, picptr, overlap
%else
    DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, tmp, sby, see, overlap
%endif

    or         overlapd, 2                  ; top_overlap: overlap & 2
    mov             r8m, overlapd
    movzx          sbyd, sbyb
%if ARCH_X86_32
    imul             r4, [fg_dataq+FGData.seed], 0x00010001
    DEFINE_ARGS tmp, src, scaling, sby, see, picptr, unused
%else
    imul           seed, [fg_dataq+FGData.seed], 0x00010001
%endif
    imul           tmpd, sbyd, 173 * 0x00010001
    imul           sbyd, 37 * 0x01000100
    add            tmpd, (105 << 16) | 188
    add            sbyd, (178 << 24) | (141 << 8)
    and            tmpd, 0x00ff00ff
    and            sbyd, 0xff00ff00
    xor            seed, tmpd
%if ARCH_X86_32
    xor            sbyd, seed               ; (cur_seed << 16) | top_seed

    DEFINE_ARGS dst, src, scaling, see, w, picptr, src_bak

    mov             r3m, seed
    mov              wq, r4m
%else
    xor            seed, sbyd               ; (cur_seed << 16) | top_seed

    DEFINE_ARGS dst, src, stride, src_bak, w, scaling, grain_lut, \
                tmp, unused2, see, unused3
%endif

    lea        src_bakq, [srcq+wq]
    neg              wq
    sub           dstmp, srcq
%if ARCH_X86_32
    mov             r1m, src_bakq
    mov             r4m, wq
    DEFINE_ARGS tmp, src, scaling, see, unused1, picptr, unused2
%endif

.loop_x_v_overlap:
%if ARCH_X86_32
    mov            seed, r3m
%endif
    ; we assume from the block above that bits 8-15 of tmpd are zero'ed,
    ; because of the 'and tmpd, 0x00ff00ff' above
    mov             r6d, seed
    or             seed, 0xeff4eff4
    test           seeb, seeh
    setp           tmpb                     ; parity of top_seed
    shr            seed, 16
    shl            tmpd, 16
    test           seeb, seeh
    setp           tmpb                     ; parity of cur_seed
    or              r6d, 0x00010001
    xor            tmpd, r6d
    mov            seed, tmpd
    ror            seed, 1                  ; updated (cur_seed << 16) | top_seed

%if ARCH_X86_32
    mov             r3m, seed

    DEFINE_ARGS dst, src, scaling, offy, unused1, unused2, offx

    mov           offxd, offyd
%else
    DEFINE_ARGS dst, src, stride, src_bak, w, scaling, grain_lut, \
                offx, offy, see, unused, top_offxy

    mov           offyd, seed
    mov           offxd, seed
%endif

    ror           offyd, 8
    ror           offxd, 12
    and           offyd, 0xf000f
    and           offxd, 0xf000f
    imul          offyd, 164
    ; offxy=offy*stride+offx, (cur_offxy << 16) | top_offxy
    lea           offyq, [offyq+offxq*2+0x10001*747+32*82]

%if ARCH_X86_32
    DEFINE_ARGS top_offxy, src, scaling, offxy, h, picptr, grain_lut
%else
    DEFINE_ARGS dst, src, stride, src_bak, w, scaling, grain_lut, \
                h, offxy, see, unused, top_offxy
%endif

    movzx    top_offxyd, offxyw
%if ARCH_X86_32
    mov [rsp+5*mmsize+1*gprsize], top_offxyd

    DEFINE_ARGS dst, src, scaling, offxy, h, picptr, grain_lut
%endif
    shr          offxyd, 16

.loop_x_odd_v_overlap:
%if ARCH_X86_32
    mov              r5, r5m
    lea              r5, [base+pb_27_17]
    mov [rsp+5*mmsize+12], r5
%else
    mova             m8, [pb_27_17]
%endif
    mov              hd, r7m
    mov      grain_lutq, grain_lutmp
.loop_y_v_overlap:
    ; src
    mova             m0, [srcq]
    pxor             m2, m2
    punpckhbw        m1, m0, m2
    punpcklbw        m0, m2                 ; m0-1: src as word

    ; scaling[src]
%if ARCH_X86_32
    vpgatherdw       m4, m0, scalingq-1, r0, r5, m3
    vpgatherdw       m5, m1, scalingq-1, r0, r5, m3
%else
    vpgatherdw       m4, m0, scalingq-1, r12, r13, m3
    vpgatherdw       m5, m1, scalingq-1, r12, r13, m3
%endif
    REPX {psrlw x, 8}, m4, m5

    ; grain = grain_lut[offy+y][offx+x]
    movu             m3, [grain_lutq+offxyq]
%if ARCH_X86_32
    mov              r5, [rsp+5*mmsize+1*gprsize]
    movu             m7, [grain_lutq+r5]
%else
    movu             m7, [grain_lutq+top_offxyq]
%endif
    punpckhbw        m6, m7, m3
    punpcklbw        m7, m3
%if ARCH_X86_32
    mov              r5, [rsp+5*mmsize+12]
    pmaddubsw        m3, [r5], m6
    pmaddubsw        m6, [r5], m7
%else
    pmaddubsw        m3, m8, m6
    pmaddubsw        m6, m8, m7
%endif
    pmulhrsw         m3, m14
    pmulhrsw         m6, m14
    packsswb         m6, m3
    pcmpgtb          m7, m2, m6
    punpcklbw        m2, m6, m7
    punpckhbw        m6, m7

    ; noise = round2(scaling[src] * grain, scaling_shift)
    pmullw           m2, m4
    pmullw           m6, m5
    pmulhrsw         m2, m11
    pmulhrsw         m6, m11

    ; dst = clip_pixel(src, noise)
    paddw            m0, m2
    paddw            m1, m6
    pmaxsw           m0, m13
    pmaxsw           m1, m13
    pminsw           m0, m12
    pminsw           m1, m12
    packuswb         m0, m1
    movifnidn      dstq, dstmp
    mova    [dstq+srcq], m0

%if ARCH_X86_32
    add dword [rsp+5*mmsize+12], mmsize
%else
    mova             m8, [pb_17_27]
%endif
    add            srcq, r2mp
    add      grain_lutq, 82
    dec              hw
    jz .end_y_v_overlap
    ; 2 lines get vertical overlap, then fall back to non-overlap code for
    ; remaining (up to) 30 lines
    btc              hd, 16
    jnc .loop_y_v_overlap
    jmp .loop_y

.end_y_v_overlap:
%if ARCH_X86_32
    add            r4mp, 16
%else
    add              wq, 16
%endif
    jge .end_hv
%if ARCH_X86_32
    mov            srcq, r1mp
    add            srcq, r4mp
%else
    lea            srcq, [src_bakq+wq]
%endif
    btc       dword r8m, 2
    jc .loop_x_hv_overlap
    add          offxyd, 16
%if ARCH_X86_32
    add dword [rsp+5*mmsize+1*gprsize], 16
%else
    add      top_offxyd, 16
%endif
    jmp .loop_x_odd_v_overlap

.loop_x_hv_overlap:
%if ARCH_X86_32
    mov              r5, r5m
    lea              r5, [base+pb_27_17]
    mov [rsp+5*mmsize+12], r5

    DEFINE_ARGS tmp, src, scaling, offxy, w, picptr, src_bak

    mov              r5, [rsp+5*mmsize+1*gprsize]
    mov              r4, offxyd
    add              r5, 16
    add              r4, 16
    mov [rsp+5*mmsize+2*gprsize], r5        ; topleft_offxy
    mov [rsp+5*mmsize+0*gprsize], r4        ; left_offxy

    DEFINE_ARGS tmp, src, scaling, see, w, picptr, src_bak

    xor            tmpd, tmpd
    mov            seed, r3m
%else
    mova             m8, [pb_27_17]

    DEFINE_ARGS dst, src, stride, src_bak, w, scaling, grain_lut, \
                tmp, unused2, see, unused3

    ; we assume from the block above that bits 8-15 of tmpd are zero'ed
%endif
    mov             r6d, seed
    or             seed, 0xeff4eff4
    test           seeb, seeh
    setp           tmpb                     ; parity of top_seed
    shr            seed, 16
    shl            tmpd, 16
    test           seeb, seeh
    setp           tmpb                     ; parity of cur_seed
    or              r6d, 0x00010001
    xor            tmpd, r6d
    mov            seed, tmpd
    ror            seed, 1                  ; updated (cur_seed << 16) | top_seed

%if ARCH_X86_32
    mov             r3m, seed

    DEFINE_ARGS dst, src, scaling, offy, unused1, unused2, offx

    mov           offxd, offyd
%else
    DEFINE_ARGS dst, src, stride, src_bak, w, scaling, grain_lut, \
                offx, offy, see, left_offxy, top_offxy, topleft_offxy

    lea  topleft_offxyq, [top_offxyq+16]
    lea     left_offxyq, [offyq+16]
    mov           offyd, seed
    mov           offxd, seed
%endif
    ror           offyd, 8
    ror           offxd, 12
    and           offyd, 0xf000f
    and           offxd, 0xf000f
    imul          offyd, 164
    ; offxy=offy*stride+offx, (cur_offxy << 16) | top_offxy
    lea           offyq, [offyq+offxq*2+0x10001*747+32*82]

%if ARCH_X86_32
    DEFINE_ARGS dst, src, scaling, offxy, h, picptr, grain_lut

    movzx            r5, offxyw             ; top_offxy
    mov [rsp+5*mmsize+1*gprsize], r5
%else
    DEFINE_ARGS dst, src, stride, src_bak, w, scaling, grain_lut, \
                h, offxy, see, left_offxy, top_offxy, topleft_offxy

    movzx    top_offxyd, offxyw
%endif
    shr          offxyd, 16

    mov              hd, r7m
    mov      grain_lutq, grain_lutmp
.loop_y_hv_overlap:
    ; grain = grain_lut[offy+y][offx+x]
    movu             m3, [grain_lutq+offxyq]
%if ARCH_X86_32
    mov              r5, [rsp+5*mmsize+1*gprsize]   ; top_offxy
    mov              r0, [rsp+5*mmsize+0*gprsize]   ; left_offxy
    movu             m6, [grain_lutq+r5]
    mov              r5, [rsp+5*mmsize+2*gprsize]   ; topleft_offxy
    movd             m4, [grain_lutq+r0]
    movd             m7, [grain_lutq+r5]
%else
    movu             m6, [grain_lutq+top_offxyq]
    movd             m4, [grain_lutq+left_offxyq]
    movd             m7, [grain_lutq+topleft_offxyq]
%endif
    ; do h interpolation first (so top | top/left -> top, left | cur -> cur)
    punpcklbw        m4, m3
    punpcklbw        m7, m6
    pmaddubsw        m2, m15, m4
    pmaddubsw        m4, m15, m7
    pmulhrsw         m2, m14
    pmulhrsw         m4, m14
    packsswb         m2, m2
    packsswb         m4, m4
    shufps           m2, m3, q3210
    shufps           m4, m6, q3210
    ; followed by v interpolation (top | cur -> cur)
    punpcklbw        m3, m4, m2
    punpckhbw        m4, m2
%if ARCH_X86_32
    mov              r5, [rsp+5*mmsize+12]
    pmaddubsw        m7, [r5], m4
    pmaddubsw        m4, [r5], m3
%else
    pmaddubsw        m7, m8, m4
    pmaddubsw        m4, m8, m3
%endif
    pmulhrsw         m7, m14
    pmulhrsw         m4, m14
    packsswb         m4, m7
    pxor             m2, m2
    pcmpgtb          m7, m2, m4
    punpcklbw        m3, m4, m7
    punpckhbw        m4, m7

    ; src
    mova             m0, [srcq]
    punpckhbw        m1, m0, m2
    punpcklbw        m0, m2                 ; m0-1: src as word

    ; scaling[src]
%if ARCH_X86_32
    vpgatherdw       m5, m0, scalingq-1, r0, r5, m7
    vpgatherdw       m6, m1, scalingq-1, r0, r5, m7
%else
    vpgatherdw       m5, m0, scalingq-1, r13, r14, m7
    vpgatherdw       m6, m1, scalingq-1, r13, r14, m7
%endif
    REPX {psrlw x, 8}, m5, m6

    ; noise = round2(scaling[src] * grain, scaling_shift)
    pmullw           m3, m5
    pmullw           m4, m6
    pmulhrsw         m3, m11
    pmulhrsw         m4, m11

    ; dst = clip_pixel(src, noise)
    paddw            m0, m3
    paddw            m1, m4
    pmaxsw           m0, m13
    pmaxsw           m1, m13
    pminsw           m0, m12
    pminsw           m1, m12
    packuswb         m0, m1
    movifnidn      dstq, dstmp
    mova    [dstq+srcq], m0

%if ARCH_X86_32
    add dword [rsp+5*mmsize+12], mmsize
%else
    mova             m8, [pb_17_27]
%endif
    add            srcq, r2mp
    add      grain_lutq, 82
    dec              hw
    jz .end_y_hv_overlap
    ; 2 lines get vertical overlap, then fall back to non-overlap code for
    ; remaining (up to) 30 lines
    btc              hd, 16
    jnc .loop_y_hv_overlap
    jmp .loop_y_h_overlap

.end_y_hv_overlap:
%if ARCH_X86_32
    add            r4mp, 16
%else
    add              wq, 16
%endif
    jge .end_hv
%if ARCH_X86_32
    mov            srcq, r1m
    add            srcq, r4m
%else
    lea            srcq, [src_bakq+wq]
%endif
    xor       dword r8m, 4
    add          offxyd, 16
%if ARCH_X86_32
    add dword [rsp+5*mmsize+1*gprsize], 16
%else
    add      top_offxyd, 16
%endif
    jmp .loop_x_odd_v_overlap

.end_hv:
    RET

%macro FGUV_FN 3 ; name, ss_hor, ss_ver
INIT_XMM ssse3
%if ARCH_X86_32
; fguv_32x32xn_i420_ssse3(dst, src, stride, fg_data, w, scaling, grain_lut, h,
;                         sby, luma, lstride, uv_pl, is_id)
%if STACK_ALIGNMENT < mmsize
DECLARE_ARG 0, 1, 2, 3, 4, 5, 6, 7, 8
cglobal fguv_32x32xn_i%1_8bpc, 0, 7, 8, 0 - (7 * mmsize + (13 + 3) * gprsize), \
        tmp, src, scaling, h, fg_data, picptr, unused
    mov              r0, r0m
    mov              r1, r2m
    mov              r2, r4m
    mov              r3, r6m
    mov              r4, r7m
    mov [rsp+7*mmsize+3*gprsize], r0
    mov [rsp+7*mmsize+5*gprsize], r1
    mov [rsp+7*mmsize+7*gprsize], r2
    mov [rsp+7*mmsize+9*gprsize], r3
    mov [rsp+7*mmsize+10*gprsize], r4

    mov              r0, r8m
    mov              r1, r9m
    mov              r2, r10m
    mov              r4, r11m
    mov              r3, r12m
    mov [rsp+7*mmsize+11*gprsize], r0
    mov [rsp+7*mmsize+12*gprsize], r1
    mov [rsp+7*mmsize+13*gprsize], r2
    mov [rsp+7*mmsize+14*gprsize], r4
%else
cglobal fguv_32x32xn_i%1_8bpc, 0, 7, 8, 7 * mmsize + (4) * gprsize, \
        tmp, src, scaling, h, fg_data, picptr, unused
%endif
    mov            srcq, srcm
    mov        fg_dataq, r3m
    mov        scalingq, r5m
%if STACK_ALIGNMENT < mmsize
%define r0m [rsp+7*mmsize+ 3*gprsize]
%define r1m [rsp+7*mmsize+ 4*gprsize]
%define r2m [rsp+7*mmsize+ 5*gprsize]
%define r3m [rsp+7*mmsize+ 6*gprsize]
%define r4m [rsp+7*mmsize+ 7*gprsize]
%define r5m [rsp+7*mmsize+ 8*gprsize]
%define r6m [rsp+7*mmsize+ 9*gprsize]
%define r7m [rsp+7*mmsize+10*gprsize]
%define r8m [rsp+7*mmsize+11*gprsize]
%define r9m [rsp+7*mmsize+12*gprsize]
%define r10m [rsp+7*mmsize+13*gprsize]
%define r11m [rsp+7*mmsize+14*gprsize]
%define r12m [rsp+7*mmsize+15*gprsize]
%endif
    LEA              r5, pb_mask
%define base r5-pb_mask
    mov             r5m, r5
%else
cglobal fguv_32x32xn_i%1_8bpc, 6, 15, 16, dst, src, stride, fg_data, w, scaling, \
                                     grain_lut, tmp, sby, luma, lstride, uv_pl, is_id
    lea              r8, [pb_mask]
%define base r8-pb_mask
%endif
    mov             r6d, [fg_dataq+FGData.scaling_shift]
    movd             m3, [base+mul_bits+r6*2-14]
    mov             r6d, [fg_dataq+FGData.clip_to_restricted_range]
    lea            tmpd, [r6d*2]
%if ARCH_X86_32 && STACK_ALIGNMENT < mmsize
    test             r3, r3
%else
    cmp      dword r12m, 0                      ; is_idm
%endif
    movd             m5, [base+min+r6*2]
    cmovne          r6d, tmpd
    movd             m4, [base+max+r6*2]
    punpcklwd        m3, m3
    punpcklwd        m5, m5
    punpcklwd        m4, m4
    pshufd           m3, m3, q0000
    pshufd           m5, m5, q0000
    pshufd           m4, m4, q0000
    SCRATCH           3, 11, 0
    SCRATCH           4, 12, 1
    SCRATCH           5, 13, 2

    cmp byte [fg_dataq+FGData.chroma_scaling_from_luma], 0
    jne .csfl

%macro %%FGUV_32x32xN_LOOP 3 ; not-csfl, ss_hor, ss_ver
%if ARCH_X86_32
    DEFINE_ARGS dst, src, scaling, sby, fg_data, picptr, overlap
%else
    DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, unused, sby, see, overlap
%endif

%if %1
    mov             r6d, dword r11m
    movd             m0, [fg_dataq+FGData.uv_mult+r6*4]
    movd             m1, [fg_dataq+FGData.uv_luma_mult+r6*4]
    punpcklbw        m6, m1, m0
    movd             m7, [fg_dataq+FGData.uv_offset+r6*4]
    punpcklwd        m6, m6
    punpcklwd        m7, m7
    pshufd           m6, m6, q0000
    pshufd           m7, m7, q0000
    SCRATCH           6, 14, 3
    SCRATCH           7, 15, 4
%endif

    mov            sbyd, r8m
    mov        overlapd, [fg_dataq+FGData.overlap_flag] ; left_overlap: overlap & 1
    test       overlapd, overlapd
    jz %%no_vertical_overlap
%if ARCH_X86_32
%if %2
    mova             m1, [base+pb_23_22_h]
%else
    mova             m1, [base+pb_27_17_17_27]
%endif
    mova             m0, [base+pw_1024]
%else
%if %2
    mova             m1, [pb_23_22_h]
%else
    mova             m1, [pb_27_17_17_27]
%endif
    mova             m0, [pw_1024]
%endif
    SCRATCH           0, 8, 5
    SCRATCH           1, 9, 6
    test           sbyd, sbyd
    jnz %%vertical_overlap
    ; fall-through

%%no_vertical_overlap:
    mov             r8m, overlapd
%if ARCH_X86_32
    DEFINE_ARGS dst, src, scaling, see, fg_data, picptr, overlap
    imul           seed, (173 << 24) | 37
%else
    imul           seed, sbyd, (173 << 24) | 37
%endif
    add            seed, (105 << 24) | 178
    rol            seed, 8
    movzx          seed, seew
    xor            seed, [fg_dataq+FGData.seed]

%if ARCH_X86_32
    mov             r3m, seed

    DEFINE_ARGS luma, src, scaling, see, w, picptr, src_bak
%define luma_bakq lumaq

    mov              wq, r4m
%if %3
    shl           r10mp, 1
%endif
%else
    DEFINE_ARGS dst, src, stride, luma, w, scaling, grain_lut, \
                unused2, unused3, see, overlap, unused4, src_bak, lstride, luma_bak

    mov        lstrideq, r10mp
%endif

    mov           lumaq, r9mp
    lea        src_bakq, [srcq+wq]
    lea       luma_bakq, [lumaq+wq*(1+%2)]
    neg              wq
    sub            r0mp, srcq
%if ARCH_X86_32
    mov             r1m, src_bakq
    mov            r11m, luma_bakq
    mov             r4m, wq

    DEFINE_ARGS tmp, src, scaling, see, unused1, picptr, unused2
%else
    mov           r11mp, src_bakq
    mov           r12mp, strideq
%endif

%%loop_x:
%if ARCH_X86_32
    mov            seed, r3m
%endif
    mov             r6d, seed
    or             seed, 0xEFF4
    shr             r6d, 1
    test           seeb, seeh
    lea            seed, [r6+0x8000]
    cmovp          seed, r6d               ; updated seed
%if ARCH_X86_32
    mov             r3m, seed

    DEFINE_ARGS dst, src, scaling, offy, w, picptr, offx

    mov           offxd, offyd
%else
    DEFINE_ARGS dst, src, stride, luma, w, scaling, grain_lut, \
                offx, offy, see, overlap, unused1, unused2, lstride

    mov           offyd, seed
    mov           offxd, seed
%endif
    ror           offyd, 8
    shr           offxd, 12
    and           offyd, 0xf
    imul          offyd, 164>>%3
    lea           offyq, [offyq+offxq*(2-%2)+(3+(6>>%3))*82+(3+(6>>%2))]  ; offy*stride+offx

%if ARCH_X86_32
    DEFINE_ARGS luma, src, scaling, offxy, h, picptr, grain_lut
%else
    DEFINE_ARGS dst, src, stride, luma, w, scaling, grain_lut, \
                h, offxy, see, overlap, unused1, unused2, lstride, luma_bak
%endif

%%loop_x_odd:
    mov              hd, r7m
    mov      grain_lutq, grain_lutmp
%%loop_y:
    ; src
%if ARCH_X86_32
    mov           lumaq, r9mp
%endif
%if %2
    mova             m4, [lumaq+ 0]
    mova             m6, [lumaq+16]
    mova             m0, [srcq]
%if ARCH_X86_32
    add           lumaq, r10mp
    mov            r9mp, lumaq
    mov              r5, r5m
    movd             m7, [base+pb_1]
%else
    movd             m7, [pb_1]
%endif
    pshufd           m7, m7, q0000
    pxor             m2, m2
    pmaddubsw        m4, m7
    pmaddubsw        m6, m7
    pavgw            m4, m2
    pavgw            m6, m2
%else
    mova             m4, [lumaq]
    mova             m0, [srcq]
%if ARCH_X86_32
    add           lumaq, r10mp
    mov            r9mp, lumaq
%endif
    pxor             m2, m2
%endif

%if %1
%if %2
    packuswb         m4, m6                 ; luma
%endif
    punpckhbw        m6, m4, m0
    punpcklbw        m4, m0                 ; { luma, chroma }
    pmaddubsw        m6, m14
    pmaddubsw        m4, m14
    psraw            m6, 6
    psraw            m4, 6
    paddw            m6, m15
    paddw            m4, m15
    packuswb         m4, m6                 ; pack+unpack = clip
    punpckhbw        m6, m4, m2
    punpcklbw        m4, m2
%elif %2 == 0
    punpckhbw        m6, m4, m2
    punpcklbw        m4, m2
%endif

    ; scaling[luma_src]
%if ARCH_X86_32
    vpgatherdw       m7, m4, scalingq-1, r0, r5
    vpgatherdw       m5, m6, scalingq-1, r0, r5
%else
    vpgatherdw       m7, m4, scalingq-1, r12, r2
    vpgatherdw       m5, m6, scalingq-1, r12, r2
%endif
    REPX {psrlw x, 8}, m7, m5

    ; unpack chroma_source
    punpckhbw        m1, m0, m2
    punpcklbw        m0, m2                 ; m0-1: src as word

    ; grain = grain_lut[offy+y][offx+x]
    movu             m3, [grain_lutq+offxyq+ 0]
    pcmpgtb          m6, m2, m3
    punpcklbw        m2, m3, m6
    punpckhbw        m3, m6

    ; noise = round2(scaling[luma_src] * grain, scaling_shift)
    pmullw           m2, m7
    pmullw           m3, m5
    pmulhrsw         m2, m11
    pmulhrsw         m3, m11

%if ARCH_X86_32
    DEFINE_ARGS dst, src, scaling, offxy, h, picptr, grain_lut
%endif

    ; dst = clip_pixel(src, noise)
    paddw            m0, m2
    paddw            m1, m3
    pmaxsw           m0, m13
    pmaxsw           m1, m13
    pminsw           m0, m12
    pminsw           m1, m12
    packuswb         m0, m1
    movifnidn      dstq, dstmp
    mova    [dstq+srcq], m0

%if ARCH_X86_32
    add            srcq, r2mp
    ; we already incremented lumaq above
%else
    add            srcq, r12mp
%if %3
    lea           lumaq, [lumaq+lstrideq*2]
%else
    add           lumaq, lstrideq
%endif
%endif
    add      grain_lutq, 82
    dec              hw
    jg %%loop_y

%if ARCH_X86_32
    DEFINE_ARGS luma, src, scaling, offxy, w, picptr, grain_lut

    mov              wq, r4m
%endif
    add              wq, 16
    jge %%end
%if ARCH_X86_32
    mov            srcq, r1mp
    mov           lumaq, r11mp
%else
    mov            srcq, r11mp
%endif
    lea           lumaq, [luma_bakq+wq*(1+%2)]
    add            srcq, wq
%if ARCH_X86_32
    mov             r4m, wq
    mov             r9m, lumaq
%endif
%if %2 == 0
    ; adjust top_offxy
%if ARCH_X86_32
    add dword [rsp+7*mmsize+1*gprsize], 16
%else
    add            r11d, 16
%endif
    add          offxyd, 16
    btc       dword r8m, 2
    jc %%loop_x_even
    test      dword r8m, 2
    jz %%loop_x_odd
    jmp %%loop_x_odd_v_overlap
%%loop_x_even:
%endif
    test      dword r8m, 1
    jz %%loop_x

    ; r8m = sbym
    test      dword r8m, 2
    jne %%loop_x_hv_overlap

    ; horizontal overlap (without vertical overlap)
%%loop_x_h_overlap:
%if ARCH_X86_32
%if %2
    lea              r6, [offxyd+16]
    mov [rsp+7*mmsize+0*gprsize], r6
%else
    mov [rsp+7*mmsize+0*gprsize], offxyd
%endif

    DEFINE_ARGS luma, src, scaling, see, w, picptr, grain_lut

    mov            seed, r3m
%else
    DEFINE_ARGS dst, src, stride, luma, w, scaling, grain_lut, \
                offx, offy, see, left_offxy, unused1, unused2, lstride

%if %2
    lea     left_offxyd, [offyd+16]         ; previous column's offy*stride+offx
%else
    mov     left_offxyd, offyd
%endif
%endif
    mov             r6d, seed
    or             seed, 0xEFF4
    shr             r6d, 1
    test           seeb, seeh
    lea            seed, [r6+0x8000]
    cmovp          seed, r6d                ; updated seed

%if ARCH_X86_32
    mov             r3m, seed

    DEFINE_ARGS luma, src, scaling, offy, w, picptr, offx

    mov          offxd, offyd
%else
    DEFINE_ARGS dst, src, stride, luma, w, scaling, grain_lut, \
                offx, offy, see, left_offxy, unused1, unused2, lstride

    mov           offyd, seed
    mov           offxd, seed
%endif
    ror           offyd, 8
    shr           offxd, 12
    and           offyd, 0xf
    imul          offyd, 164>>%3
    lea           offyq, [offyq+offxq*(2-%2)+(3+(6>>%3))*82+3+(6>>%2)]  ; offy*stride+offx

%if ARCH_X86_32
    DEFINE_ARGS luma, src, scaling, offxy, h, picptr, grain_lut
%else
    DEFINE_ARGS dst, src, stride, luma, w, scaling, grain_lut, \
                h, offxy, see, left_offxy, unused1, unused2, lstride, luma_bak
%endif

    mov              hd, r7m
    mov      grain_lutq, grain_lutmp
%%loop_y_h_overlap:
    ; src
%if ARCH_X86_32
    mov           lumaq, r9mp
%endif
%if %2
    mova             m4, [lumaq+ 0]
    mova             m6, [lumaq+16]
    mova             m0, [srcq]
%if ARCH_X86_32
    add           lumaq, r10mp
    mov            r9mp, lumaq
    mov              r5, r5m
    movd             m7, [base+pb_1]
%else
    movd             m7, [pb_1]
%endif
    pshufd           m7, m7, q0000
    pxor             m2, m2
    pmaddubsw        m4, m7
    pmaddubsw        m6, m7
    pavgw            m4, m2
    pavgw            m6, m2
%else
    mova             m4, [lumaq]
    mova             m0, [srcq]
%if ARCH_X86_32
    add           lumaq, r10mp
    mov            r9mp, lumaq
%endif
    pxor             m2, m2
%endif

%if %1
%if %2
    packuswb         m4, m6                 ; luma
%endif
    punpckhbw        m6, m4, m0
    punpcklbw        m4, m0                 ; { luma, chroma }
    pmaddubsw        m6, m14
    pmaddubsw        m4, m14
    psraw            m6, 6
    psraw            m4, 6
    paddw            m6, m15
    paddw            m4, m15
    packuswb         m4, m6                 ; pack+unpack = clip
    punpckhbw        m6, m4, m2
    punpcklbw        m4, m2
%elif %2 == 0
    punpckhbw        m6, m4, m2
    punpcklbw        m4, m2
%endif

    ; scaling[luma_src]
%if ARCH_X86_32
    vpgatherdw       m7, m4, scalingq-1, r0, r5
    vpgatherdw       m5, m6, scalingq-1, r0, r5
%else
    vpgatherdw       m7, m4, scalingq-1, r12, r2
    vpgatherdw       m5, m6, scalingq-1, r12, r2
%endif
    REPX {psrlw x, 8}, m7, m5

    ; unpack chroma_source
    punpckhbw        m1, m0, m2
    punpcklbw        m0, m2                 ; m0-1: src as word

    ; grain = grain_lut[offy+y][offx+x]
    movu             m4, [grain_lutq+offxyq+ 0]
%if ARCH_X86_32
    mov              r0, [rsp+7*mmsize+0*gprsize]
    movd             m2, [grain_lutq+r0+ 0]
%else
    movd             m2, [grain_lutq+left_offxyq+ 0]
%endif
    punpcklbw        m2, m4
    pmaddubsw        m3, m9, m2
    pmulhrsw         m3, m8
    packsswb         m3, m3
    shufps           m3, m4, q3210
    pxor             m4, m4
    pcmpgtb          m4, m3
    punpcklbw        m2, m3, m4
    punpckhbw        m3, m4

    ; noise = round2(scaling[luma_src] * grain, scaling_shift)
    pmullw           m2, m7
    pmullw           m3, m5
    pmulhrsw         m2, m11
    pmulhrsw         m3, m11

%if ARCH_X86_32
    DEFINE_ARGS dst, src, scaling, offxy, h, picptr, grain_lut
%endif

    ; dst = clip_pixel(src, noise)
    paddw            m0, m2
    paddw            m1, m3
    pmaxsw           m0, m13
    pmaxsw           m1, m13
    pminsw           m0, m12
    pminsw           m1, m12
    packuswb         m0, m1
    movifnidn      dstq, dstmp
    mova    [dstq+srcq], m0

%if ARCH_X86_32
    add            srcq, r2mp
    ; lumaq has already been incremented above
%else
    add            srcq, r12mp
%if %3
    lea           lumaq, [lumaq+lstrideq*2]
%else
    add           lumaq, lstrideq
%endif
%endif
    add      grain_lutq, 82
    dec              hw
    jg %%loop_y_h_overlap

%if ARCH_X86_32
    DEFINE_ARGS luma, src, scaling, offxy, w, picptr, grain_lut

    mov              wq, r4m
%endif
    add              wq, 16
    jge %%end
%if ARCH_X86_32
    mov            srcq, r1mp
    mov           lumaq, r11mp
%else
    mov            srcq, r11mp
%endif
    lea           lumaq, [luma_bakq+wq*(1+%2)]
    add            srcq, wq
%if ARCH_X86_32
    mov             r4m, wq
    mov             r9m, lumaq
%endif
%if %2 == 0
    xor       dword r8m, 4
    ; adjust top_offxyd
%if ARCH_X86_32
    add dword [rsp+7*mmsize+1*gprsize], 16
%else
    add            r11d, 16
%endif
    add          offxyd, 16
%endif

    ; r8m = sbym
    test      dword r8m, 2
%if %2
    jne %%loop_x_hv_overlap
    jmp %%loop_x_h_overlap
%else
    jne %%loop_x_odd_v_overlap
    jmp %%loop_x_odd
%endif

%%end:
    RET

%%vertical_overlap:
%if ARCH_X86_32
    DEFINE_ARGS dst, src, scaling, sby, fg_data, picptr, overlap
%else
    DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, tmp, sby, see, overlap
%endif

    or         overlapd, 2                  ; top_overlap: overlap & 2
    mov             r8m, overlapd
    movzx          sbyd, sbyb
%if ARCH_X86_32
    imul             r4, [fg_dataq+FGData.seed], 0x00010001
    DEFINE_ARGS tmp, src, scaling, sby, see, picptr, unused
%else
    imul           seed, [fg_dataq+FGData.seed], 0x00010001
%endif
    imul           tmpd, sbyd, 173 * 0x00010001
    imul           sbyd, 37 * 0x01000100
    add            tmpd, (105 << 16) | 188
    add            sbyd, (178 << 24) | (141 << 8)
    and            tmpd, 0x00ff00ff
    and            sbyd, 0xff00ff00
    xor            seed, tmpd
%if ARCH_X86_32
    xor            sbyd, seed               ; (cur_seed << 16) | top_seed

    DEFINE_ARGS luma, src, scaling, see, w, picptr, src_bak

    mov             r3m, seed
    mov              wq, r4m
%if %3
    shl           r10mp, 1
%endif
%else
    xor            seed, sbyd               ; (cur_seed << 16) | top_seed

    DEFINE_ARGS dst, src, stride, luma, w, scaling, grain_lut, \
                tmp, unused2, see, overlap, unused3, src_bak, lstride, luma_bak

    mov        lstrideq, r10mp
%endif

    mov           lumaq, r9mp
    lea        src_bakq, [srcq+wq]
    lea       luma_bakq, [lumaq+wq*(1+%2)]
    neg              wq
    sub            r0mp, srcq
%if ARCH_X86_32
    mov             r1m, src_bakq
    mov            r11m, luma_bakq
    mov             r4m, wq

    DEFINE_ARGS tmp, src, scaling, see, unused1, picptr, unused2
%else
    mov           r11mp, src_bakq
    mov           r12mp, strideq
%endif

%%loop_x_v_overlap:
%if ARCH_X86_32
    mov            seed, r3m
    xor            tmpd, tmpd
%endif
    ; we assume from the block above that bits 8-15 of tmpd are zero'ed
    mov             r6d, seed
    or             seed, 0xeff4eff4
    test           seeb, seeh
    setp           tmpb                     ; parity of top_seed
    shr            seed, 16
    shl            tmpd, 16
    test           seeb, seeh
    setp           tmpb                     ; parity of cur_seed
    or              r6d, 0x00010001
    xor            tmpd, r6d
    mov            seed, tmpd
    ror            seed, 1                  ; updated (cur_seed << 16) | top_seed

%if ARCH_X86_32
    mov             r3m, seed

    DEFINE_ARGS dst, src, scaling, offy, h, picptr, offx

    mov           offxd, offyd
%else
    DEFINE_ARGS dst, src, stride, luma, w, scaling, grain_lut, \
                offx, offy, see, overlap, top_offxy, unused, lstride

    mov           offxd, seed
    mov           offyd, seed
%endif
    ror           offyd, 8
    ror           offxd, 12
    and           offyd, 0xf000f
    and           offxd, 0xf000f
    imul          offyd, 164>>%3
    ; offxy=offy*stride+offx, (cur_offxy << 16) | top_offxy
    lea           offyq, [offyq+offxq*(2-%2)+0x10001*((3+(6>>%3))*82+3+(6>>%2))+(32>>%3)*82]

%if ARCH_X86_32
    DEFINE_ARGS tmp, src, scaling, offxy, h, picptr, top_offxy
%else
    DEFINE_ARGS dst, src, stride, luma, w, scaling, grain_lut, \
                h, offxy, see, overlap, top_offxy, unused, lstride, luma_bak
%endif

    movzx    top_offxyd, offxyw
    shr          offxyd, 16
%if ARCH_X86_32
    mov [rsp+7*mmsize+1*gprsize], top_offxyd

    DEFINE_ARGS luma, src, scaling, offxy, h, picptr, grain_lut
%endif

%%loop_x_odd_v_overlap:
    mov              hd, r7m
    mov      grain_lutq, grain_lutmp
%if ARCH_X86_32
    mov              r5, r5m
%endif
%if %3
    mova             m1, [PIC_ptr(pb_23_22)]
%else
    mova             m1, [PIC_ptr(pb_27_17)]
%endif
%%loop_y_v_overlap:
%if ARCH_X86_32
    mov           lumaq, r9mp
%endif
%if %2
    mova             m4, [lumaq+ 0]
    mova             m6, [lumaq+16]
    mova             m0, [srcq]
%if ARCH_X86_32
    add           lumaq, r10mp
    mov            r9mp, lumaq
    mov              r5, r5m
    movd             m7, [base+pb_1]
%else
    movd             m7, [pb_1]
%endif
    pshufd           m7, m7, q0000
    pxor             m2, m2
    pmaddubsw        m4, m7
    pmaddubsw        m6, m7
    pavgw            m4, m2
    pavgw            m6, m2
%else
    mova             m4, [lumaq]
    mova             m0, [srcq]
%if ARCH_X86_32
    add           lumaq, r10mp
    mov            r9mp, lumaq
%endif
    pxor             m2, m2
%endif

%if %1
%if %2
    packuswb         m4, m6                 ; luma
%endif
    punpckhbw        m6, m4, m0
    punpcklbw        m4, m0                 ; { luma, chroma }
    pmaddubsw        m6, m14
    pmaddubsw        m4, m14
    psraw            m6, 6
    psraw            m4, 6
    paddw            m6, m15
    paddw            m4, m15
    packuswb         m4, m6                 ; pack+unpack = clip
    punpckhbw        m6, m4, m2
    punpcklbw        m4, m2
%elif %2 == 0
    punpckhbw        m6, m4, m2
    punpcklbw        m4, m2
%endif

    ; scaling[luma_src]
%if ARCH_X86_32
    vpgatherdw       m7, m4, scalingq-1, r0, r5
    vpgatherdw       m5, m6, scalingq-1, r0, r5
%else
    vpgatherdw       m7, m4, scalingq-1, r12, r2
    vpgatherdw       m5, m6, scalingq-1, r12, r2
%endif
    REPX {psrlw x, 8}, m7, m5

    ; grain = grain_lut[offy+y][offx+x]
    movu             m3, [grain_lutq+offxyq]
%if ARCH_X86_32
    mov              r0, [rsp+7*mmsize+1*gprsize]
    movu             m4, [grain_lutq+r0]
%else
    movu             m4, [grain_lutq+top_offxyq]
%endif
    punpckhbw        m6, m4, m3
    punpcklbw        m4, m3
    pmaddubsw        m2, m1, m6
    pmaddubsw        m3, m1, m4
    pmulhrsw         m2, m8
    pmulhrsw         m3, m8
    packsswb         m3, m2
    pxor             m6, m6
    pcmpgtb          m6, m3
    punpcklbw        m2, m3, m6
    punpckhbw        m3, m6

    ; noise = round2(scaling[luma_src] * grain, scaling_shift)
    pmullw           m2, m7
    pmullw           m3, m5
    pmulhrsw         m2, m11
    pmulhrsw         m3, m11

    ; unpack chroma_source
    pxor             m4, m4
    punpckhbw        m6, m0, m4
    punpcklbw        m0, m4                 ; m0-1: src as word

%if ARCH_X86_32
    DEFINE_ARGS dst, src, scaling, offxy, h, picptr, grain_lut
%endif

    ; dst = clip_pixel(src, noise)
    paddw            m0, m2
    paddw            m6, m3
    pmaxsw           m0, m13
    pmaxsw           m6, m13
    pminsw           m0, m12
    pminsw           m6, m12
    packuswb         m0, m6
    movifnidn      dstq, dstmp
    mova    [dstq+srcq], m0

    dec              hw
    je %%end_y_v_overlap
%if ARCH_X86_32
    add            srcq, r2mp
    ; lumaq has already been incremented above
%else
    add            srcq, r12mp
%if %3
    lea           lumaq, [lumaq+lstrideq*2]
%else
    add           lumaq, lstrideq
%endif
%endif
    add      grain_lutq, 82
%if %3 == 0
    btc              hd, 16
%if ARCH_X86_32
    mov              r5, r5m
%endif
    mova             m1, [PIC_ptr(pb_17_27)]
    jnc %%loop_y_v_overlap
%endif
    jmp %%loop_y

%%end_y_v_overlap:
%if ARCH_X86_32
    DEFINE_ARGS luma, src, scaling, offxy, w, picptr, grain_lut

    mov              wq, r4m
%endif
    add              wq, 16
    jge %%end_hv
%if ARCH_X86_32
    mov            srcq, r1mp
    mov           lumaq, r11mp
%else
    mov            srcq, r11mp
%endif
    lea           lumaq, [luma_bakq+wq*(1+%2)]
    add            srcq, wq
%if ARCH_X86_32
    mov             r4m, wq
    mov             r9m, lumaq
%endif

%if %2
    ; since fg_dataq.overlap is guaranteed to be set, we never jump
    ; back to .loop_x_v_overlap, and instead always fall-through to
    ; h+v overlap
%else
%if ARCH_X86_32
    add dword [rsp+7*mmsize+1*gprsize], 16
%else
    add      top_offxyd, 16
%endif
    add          offxyd, 16
    btc       dword r8m, 2
    jnc %%loop_x_odd_v_overlap
%endif

%%loop_x_hv_overlap:
%if ARCH_X86_32
    DEFINE_ARGS tmp, src, scaling, offxy, w, picptr, unused

    mov              r6, [rsp+7*mmsize+1*gprsize]
%if %2
    lea              r0, [r3d+16]
    add              r6, 16
    mov [rsp+7*mmsize+0*gprsize], r0        ; left_offxy
%else
    mov [rsp+7*mmsize+0*gprsize], r3        ; left_offxy
%endif
    mov [rsp+7*mmsize+2*gprsize], r6        ; topleft_offxy

    DEFINE_ARGS tmp, src, scaling, see, w, picptr, unused

    mov            seed, r3m
    xor            tmpd, tmpd
%else
    DEFINE_ARGS dst, src, stride, luma, w, scaling, grain_lut, \
                tmp, offxy, see, left_offxy, top_offxy, topleft_offxy, lstride

%if %2
    lea  topleft_offxyq, [top_offxyq+16]
    lea     left_offxyq, [offxyq+16]
%else
    mov  topleft_offxyq, top_offxyq
    mov     left_offxyq, offxyq
%endif

    ; we assume from the block above that bits 8-15 of tmpd are zero'ed
%endif
    mov             r6d, seed
    or             seed, 0xeff4eff4
    test           seeb, seeh
    setp           tmpb                     ; parity of top_seed
    shr            seed, 16
    shl            tmpd, 16
    test           seeb, seeh
    setp           tmpb                     ; parity of cur_seed
    or              r6d, 0x00010001
    xor            tmpd, r6d
    mov            seed, tmpd
    ror            seed, 1                  ; updated (cur_seed << 16) | top_seed

%if ARCH_X86_32
    mov             r3m, seed

    DEFINE_ARGS tmp, src, scaling, offy, w, picptr, offx

    mov           offxd, offyd
%else
    DEFINE_ARGS dst, src, stride, luma, w, scaling, grain_lut, \
                offx, offy, see, left_offxy, top_offxy, topleft_offxy, lstride

    mov           offxd, seed
    mov           offyd, seed
%endif
    ror           offyd, 8
    ror           offxd, 12
    and           offyd, 0xf000f
    and           offxd, 0xf000f
    imul          offyd, 164>>%3
    ; offxy=offy*stride+offx, (cur_offxy << 16) | top_offxy
    lea           offyq, [offyq+offxq*(2-%2)+0x10001*((3+(6>>%3))*82+3+(6>>%2))+(32>>%3)*82]

%if ARCH_X86_32
    DEFINE_ARGS top_offxy, src, scaling, offxy, h, picptr, grain_lut
%else
    DEFINE_ARGS dst, src, stride, luma, w, scaling, grain_lut, \
                h, offxy, see, left_offxy, top_offxy, topleft_offxy, lstride, luma_bak
%endif

    movzx    top_offxyd, offxyw
    shr          offxyd, 16
%if ARCH_X86_32
    mov [rsp+7*mmsize+1*gprsize], top_offxyd
%endif

    mov              hd, r7m
    mov      grain_lutq, grain_lutmp
%if ARCH_X86_32
    mov              r5, r5m
%endif
%if %3
    mova             m3, [PIC_ptr(pb_23_22)]
%else
    mova             m3, [PIC_ptr(pb_27_17)]
%endif
%%loop_y_hv_overlap:
    ; grain = grain_lut[offy+y][offx+x]
%if ARCH_X86_32
    mov              r0, [rsp+7*mmsize+2*gprsize]       ; topleft_offxy
    mov              r5, [rsp+7*mmsize+1*gprsize]       ; top_offxy
    movd             m1, [grain_lutq+r0]
    mov              r0, [rsp+7*mmsize+0*gprsize]       ; left_offxy
%else
    movd             m1, [grain_lutq+topleft_offxyq]
%endif
    movu             m2, [grain_lutq+offxyq]
%if ARCH_X86_32
    movu             m6, [grain_lutq+r5]
    movd             m4, [grain_lutq+r0]
%else
    movu             m6, [grain_lutq+top_offxyq]
    movd             m4, [grain_lutq+left_offxyq]
%endif
    ; do h interpolation first (so top | top/left -> top, left | cur -> cur)
    punpcklbw        m1, m6
    punpcklbw        m4, m2
    pmaddubsw        m0, m9, m1
    pmaddubsw        m1, m9, m4
    REPX {pmulhrsw x, m8}, m0, m1
    packsswb         m0, m1
    shufps           m4, m0, m2, q3232
    shufps           m0, m6, q3210
    ; followed by v interpolation (top | cur -> cur)
    punpcklbw        m2, m0, m4
    punpckhbw        m0, m4
    pmaddubsw        m4, m3, m0
    pmaddubsw        m1, m3, m2
    pmulhrsw         m4, m8
    pmulhrsw         m1, m8
    packsswb         m1, m4

    ; src
%if ARCH_X86_32
    DEFINE_ARGS luma, src, scaling, offxy, w, picptr, grain_lut

    mov           lumaq, r9mp
%endif
%if %2
    mova             m4, [lumaq+ 0]
    mova             m6, [lumaq+16]
    mova             m0, [srcq]
%if ARCH_X86_32
    add           lumaq, r10mp
    mov            r9mp, lumaq
    mov              r5, r5m
    movd             m7, [base+pb_1]
%else
    movd             m7, [pb_1]
%endif
    pshufd           m7, m7, q0000
    pxor             m2, m2
    pmaddubsw        m4, m7
    pmaddubsw        m6, m7
    pavgw            m4, m2
    pavgw            m6, m2
%else
    mova             m4, [lumaq]
    mova             m0, [srcq]
%if ARCH_X86_32
    add           lumaq, r10mp
    mov            r9mp, lumaq
%endif
    pxor             m2, m2
%endif

%if %1
%if %2
    packuswb         m4, m6                 ; luma
%endif
    punpckhbw        m6, m4, m0
    punpcklbw        m4, m0                 ; { luma, chroma }
    pmaddubsw        m6, m14
    pmaddubsw        m4, m14
    psraw            m6, 6
    psraw            m4, 6
    paddw            m6, m15
    paddw            m4, m15
    packuswb         m4, m6                 ; pack+unpack = clip
    punpckhbw        m6, m4, m2
    punpcklbw        m4, m2
%elif %2 == 0
    punpckhbw        m6, m4, m2
    punpcklbw        m4, m2
%endif

    ; scaling[src]
%if ARCH_X86_32
    vpgatherdw       m7, m4, scalingq-1, r0, r5
    vpgatherdw       m5, m6, scalingq-1, r0, r5
%else
%if %3
    vpgatherdw       m7, m4, scalingq-1, r2, r12
    vpgatherdw       m5, m6, scalingq-1, r2, r12
%else
    vpgatherdw       m7, m4, scalingq-1, r2, r13
    vpgatherdw       m5, m6, scalingq-1, r2, r13
%endif
%endif
    REPX {psrlw x, 8}, m7, m5

    ; unpack grain
    pxor             m4, m4
    pcmpgtb          m4, m1
    punpcklbw        m2, m1, m4
    punpckhbw        m1, m4

    ; noise = round2(scaling[src] * grain, scaling_shift)
    pmullw           m2, m7
    pmullw           m1, m5
    pmulhrsw         m2, m11
    pmulhrsw         m1, m11

%if ARCH_X86_32
    DEFINE_ARGS dst, src, scaling, offxy, h, picptr, grain_lut
%endif

    ; unpack chroma source
    pxor             m4, m4
    punpckhbw        m5, m0, m4
    punpcklbw        m0, m4                 ; m0-1: src as word

    ; dst = clip_pixel(src, noise)
    paddw            m0, m2
    paddw            m5, m1
    pmaxsw           m0, m13
    pmaxsw           m5, m13
    pminsw           m0, m12
    pminsw           m5, m12
    packuswb         m0, m5
    movifnidn      dstq, dstmp
    mova    [dstq+srcq], m0

%if ARCH_X86_32
    add            srcq, r2mp
    ; lumaq has been adjusted above already
%else
    add            srcq, r12mp
%if %3
    lea           lumaq, [lumaq+lstrideq*(1+%2)]
%else
    add           lumaq, r10mp
%endif
%endif
    add      grain_lutq, 82
    dec              hw
%if %3
    jg %%loop_y_h_overlap
%else
    jle %%end_y_hv_overlap
%if ARCH_X86_32
    mov              r5, r5m
%endif
    mova             m3, [PIC_ptr(pb_17_27)]
    btc              hd, 16
    jnc %%loop_y_hv_overlap
%if ARCH_X86_64
    mov        lstrideq, r10mp
%endif
    jmp %%loop_y_h_overlap
%%end_y_hv_overlap:
%if ARCH_X86_64
    mov        lstrideq, r10mp
%endif
%endif

%if ARCH_X86_32
    DEFINE_ARGS luma, src, scaling, offxy, w, picptr, grain_lut

    mov              wq, r4m
%endif
    add              wq, 16
    jge %%end_hv
%if ARCH_X86_32
    mov            srcq, r1mp
    mov           lumaq, r11mp
%else
    mov            srcq, r11mp
%endif
    lea           lumaq, [luma_bakq+wq*(1+%2)]
    add            srcq, wq
%if ARCH_X86_32
    mov             r4m, wq
    mov             r9m, lumaq
%endif
%if %2
    jmp %%loop_x_hv_overlap
%else
%if ARCH_X86_32
    add dword [rsp+7*mmsize+1*gprsize], 16
%else
    add      top_offxyd, 16
%endif
    add          offxyd, 16
    xor       dword r8m, 4
    jmp %%loop_x_odd_v_overlap
%endif

%%end_hv:
    RET
%endmacro

    %%FGUV_32x32xN_LOOP 1, %2, %3
.csfl:
    %%FGUV_32x32xN_LOOP 0, %2, %3
%endmacro

FGUV_FN 420, 1, 1

%if STACK_ALIGNMENT < mmsize
DECLARE_ARG 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12
%endif

FGUV_FN 422, 1, 0

%if STACK_ALIGNMENT < mmsize
DECLARE_ARG 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12
%endif

FGUV_FN 444, 0, 0