path: root/third_party/dav1d/src/x86/looprestoration_avx512.asm

; Copyright © 2021, VideoLAN and dav1d authors
; Copyright © 2021, Two Orioles, LLC
; All rights reserved.
;
; Redistribution and use in source and binary forms, with or without
; modification, are permitted provided that the following conditions are met:
;
; 1. Redistributions of source code must retain the above copyright notice, this
;    list of conditions and the following disclaimer.
;
; 2. Redistributions in binary form must reproduce the above copyright notice,
;    this list of conditions and the following disclaimer in the documentation
;    and/or other materials provided with the distribution.
;
; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
; ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
; WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
; ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
; (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
; ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
; SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

%include "config.asm"
%include "ext/x86/x86inc.asm"

%if ARCH_X86_64

SECTION_RODATA 32

wiener_shufA:  db  1,  2,  7,  6,  3,  4,  9,  8,  5,  6, 11, 10,  7,  8, 13, 12
wiener_shufB:  db  2,  3,  8,  7,  4,  5, 10,  9,  6,  7, 12, 11,  8,  9, 14, 13
wiener_shufC:  db  3,  4,  4,  5,  5,  6,  6,  7,  7,  8,  8,  9,  9, 10, 10, 11
wiener_shufD:  db  4,  5,  5,  6,  6,  7,  7,  8,  8,  9,  9, 10, 10, 11, 11, 12
wiener_perm32: db  1,  9,  3, 11,  5, 13,  7, 15, 33, 41, 35, 43, 37, 45, 39, 47
               db 17, 25, 19, 27, 21, 29, 23, 31, 49, 57, 51, 59, 53, 61, 55, 63
sgr_shuf:      db 128, 1, -1,  2,132,  3, -1,  4,136,  5, -1,  6,140,  7, -1,  8
               db 129, 9, -1, 10,133, 11, -1, 12,137, -1, -1, -1,141, -1,  0,128
sgr_mix_perm:  db  1,  3,  5,  7, 17, 19, 21, 23, 33, 35, 37, 39, 49, 51, 53, 55
r_ext_mask:    times 68 db -1
               times  4 db  0
wiener_x_shuf: db  0,  2, -1,  0
wiener_x_add:  db  0,  1,127,  0

pw_61448:      times 2 dw 61448
pw_164_455:    dw 164, 455
pd_m16380:     dd -16380
pd_m4096:      dd -4096
pd_m25         dd -25
pd_m9:         dd -9
pd_34816:      dd 34816
pd_8421376:    dd 8421376

cextern sgr_x_by_x

SECTION .text

DECLARE_REG_TMP 8, 7, 9, 11, 12, 13, 14 ; ring buffer pointers

INIT_ZMM avx512icl
cglobal wiener_filter7_8bpc, 4, 15, 20, -384*12-16, dst, stride, left, lpf, \
                                                    w, h, edge, flt
    mov           fltq, r6mp
    mov             wd, wm
    movifnidn       hd, hm
    mov          edged, r7m
    vbroadcasti32x4 m6, [wiener_shufA]
    vbroadcasti32x4 m7, [wiener_shufB]
    mov           r10d, 0xfffe
    vbroadcasti32x4 m8, [wiener_shufC]
    vbroadcasti32x4 m9, [wiener_shufD]
    kmovw           k1, r10d
    vpbroadcastd    m0, [wiener_x_shuf]
    vpbroadcastd    m1, [wiener_x_add]
    mov            r10, 0xaaaaaaaaaaaaaaaa
    vpbroadcastd   m11, [fltq+ 0]
    vpbroadcastd   m12, [fltq+ 4]
    kmovq           k2, r10
    vpbroadcastd   m10, [pd_m16380]
    packsswb       m11, m11 ; x0   x1   x0   x1
    vpbroadcastd   m14, [fltq+16]
    pshufb         m12, m0
    vpbroadcastd   m15, [fltq+20]
    paddb          m12, m1  ; x2   x3+1 x2   127
    vpbroadcastd   m13, [pd_8421376]
    psllw          m14, 5   ; y0 y1
    psllw          m15, 5   ; y2 y3
    cmp             wd, 32  ; the minimum lr unit size for chroma in 4:2:0 is 32
    jle .w32                ; pixels, so we need a special case for small widths
    lea             t1, [rsp+wq*2+16]
    add           lpfq, wq
    add           dstq, wq
    neg             wq
    test         edgeb, 4 ; LR_HAVE_TOP
    jz .no_top
    call .h_top
    add           lpfq, strideq
    mov             t6, t1
    mov             t5, t1
    add             t1, 384*2
    call .h_top
    lea            r10, [lpfq+strideq*4]
    mov           lpfq, dstq
    mov             t4, t1
    add             t1, 384*2
    add            r10, strideq
    mov          [rsp], r10 ; below
    call .h
    mov             t3, t1
    mov             t2, t1
    dec             hd
    jz .v1
    add           lpfq, strideq
    add             t1, 384*2
    call .h
    mov             t2, t1
    dec             hd
    jz .v2
    add           lpfq, strideq
    add             t1, 384*2
    call .h
    dec             hd
    jz .v3
.main:
    lea             t0, [t1+384*2]
.main_loop:
    call .hv
    dec             hd
    jnz .main_loop
    test         edgeb, 8 ; LR_HAVE_BOTTOM
    jz .v3
    mov           lpfq, [rsp]
    call .hv_bottom
    add           lpfq, strideq
    call .hv_bottom
.v1:
    call .v
    RET
.no_top:
    lea            r10, [lpfq+strideq*4]
    mov           lpfq, dstq
    lea            r10, [r10+strideq*2]
    mov          [rsp], r10
    call .h
    mov             t6, t1
    mov             t5, t1
    mov             t4, t1
    mov             t3, t1
    mov             t2, t1
    dec             hd
    jz .v1
    add           lpfq, strideq
    add             t1, 384*2
    call .h
    mov             t2, t1
    dec             hd
    jz .v2
    add           lpfq, strideq
    add             t1, 384*2
    call .h
    dec             hd
    jz .v3
    lea             t0, [t1+384*2]
    call .hv
    dec             hd
    jz .v3
    add             t0, 384*8
    call .hv
    dec             hd
    jnz .main
.v3:
    call .v
.v2:
    call .v
    jmp .v1
.h:
    mov            r10, wq
    test         edgeb, 1 ; LR_HAVE_LEFT
    jz .h_extend_left
    movd          xm16, [leftq]
    vmovdqu32  m16{k1}, [lpfq+r10-4]
    add          leftq, 4
    jmp .h_main
.h_extend_left:
    vpbroadcastb  xm16, [lpfq+r10]   ; the masked load ensures that no exception
    vmovdqu32  m16{k1}, [lpfq+r10-4] ; gets raised from accessing invalid memory
    jmp .h_main
.h_top:
    mov            r10, wq
    test         edgeb, 1 ; LR_HAVE_LEFT
    jz .h_extend_left
.h_loop:
    movu           m16, [lpfq+r10-4]
.h_main:
    movu           m17, [lpfq+r10+4]
    test         edgeb, 2 ; LR_HAVE_RIGHT
    jnz .h_have_right
    cmp           r10d, -66
    jl .h_have_right
    push            r0
    lea             r0, [r_ext_mask+65]
    vpbroadcastb    m0, [lpfq-1]
    vpternlogd     m16, m0, [r0+r10+0], 0xe4 ; c ? a : b
    vpternlogd     m17, m0, [r0+r10+8], 0xe4
    pop             r0
.h_have_right:
    pshufb          m4, m16, m6
    mova            m0, m10
    vpdpbusd        m0, m4, m11
    pshufb          m4, m16, m7
    mova            m2, m10
    vpdpbusd        m2, m4, m11
    pshufb          m4, m17, m6
    mova            m1, m10
    vpdpbusd        m1, m4, m11
    pshufb          m4, m17, m7
    mova            m3, m10
    vpdpbusd        m3, m4, m11
    pshufb          m4, m16, m8
    vpdpbusd        m0, m4, m12
    pshufb         m16, m9
    vpdpbusd        m2, m16, m12
    pshufb          m4, m17, m8
    vpdpbusd        m1, m4, m12
    pshufb         m17, m9
    vpdpbusd        m3, m17, m12
    packssdw        m0, m2
    packssdw        m1, m3
    psraw           m0, 3
    psraw           m1, 3
    mova [t1+r10*2+ 0], m0
    mova [t1+r10*2+64], m1
    add            r10, 64
    jl .h_loop
    ret
ALIGN function_align
.hv:
    add           lpfq, strideq
    mov            r10, wq
    test         edgeb, 1 ; LR_HAVE_LEFT
    jz .hv_extend_left
    movd          xm16, [leftq]
    vmovdqu32  m16{k1}, [lpfq+r10-4]
    add          leftq, 4
    jmp .hv_main
.hv_extend_left:
    vpbroadcastb  xm16, [lpfq+r10]
    vmovdqu32  m16{k1}, [lpfq+r10-4]
    jmp .hv_main
.hv_bottom:
    mov            r10, wq
    test         edgeb, 1 ; LR_HAVE_LEFT
    jz .hv_extend_left
.hv_loop:
    movu           m16, [lpfq+r10-4]
.hv_main:
    movu           m17, [lpfq+r10+4]
    test         edgeb, 2 ; LR_HAVE_RIGHT
    jnz .hv_have_right
    cmp           r10d, -66
    jl .hv_have_right
    push            r0
    lea             r0, [r_ext_mask+65]
    vpbroadcastb    m0, [lpfq-1]
    vpternlogd     m16, m0, [r0+r10+0], 0xe4 ; c ? a : b
    vpternlogd     m17, m0, [r0+r10+8], 0xe4
    pop             r0
.hv_have_right:
    pshufb          m4, m16, m6
    mova            m0, m10
    vpdpbusd        m0, m4, m11
    pshufb          m4, m16, m7
    mova            m2, m10
    vpdpbusd        m2, m4, m11
    pshufb          m4, m17, m6
    mova            m1, m10
    vpdpbusd        m1, m4, m11
    pshufb          m4, m17, m7
    mova            m3, m10
    vpdpbusd        m3, m4, m11
    pshufb          m4, m16, m8
    vpdpbusd        m0, m4, m12
    pshufb         m16, m9
    vpdpbusd        m2, m16, m12
    pshufb          m4, m17, m8
    vpdpbusd        m1, m4, m12
    pshufb         m17, m9
    vpdpbusd        m3, m17, m12
    packssdw        m0, m2
    packssdw        m1, m3
    psraw           m0, 3
    psraw           m1, 3
    mova           m16, [t4+r10*2]
    paddw          m16, [t2+r10*2]
    mova            m3, [t3+r10*2]
    mova           m17, [t4+r10*2+64]
    paddw          m17, [t2+r10*2+64]
    mova            m5, [t3+r10*2+64]
    punpcklwd       m4, m16, m3
    mova            m2, m13
    vpdpwssd        m2, m4, m15
    punpcklwd      m18, m17, m5
    mova            m4, m13
    vpdpwssd        m4, m18, m15
    punpckhwd      m16, m3
    mova            m3, m13
    vpdpwssd        m3, m16, m15
    punpckhwd      m17, m5
    mova            m5, m13
    vpdpwssd        m5, m17, m15
    mova           m17, [t5+r10*2]
    paddw          m17, [t1+r10*2]
    paddw          m16, m0, [t6+r10*2]
    mova           m19, [t5+r10*2+64]
    paddw          m19, [t1+r10*2+64]
    paddw          m18, m1, [t6+r10*2+64]
    mova [t0+r10*2+ 0], m0
    mova [t0+r10*2+64], m1
    punpcklwd       m0, m16, m17
    vpdpwssd        m2, m0, m14
    punpcklwd       m1, m18, m19
    vpdpwssd        m4, m1, m14
    punpckhwd      m16, m17
    vpdpwssd        m3, m16, m14
    punpckhwd      m18, m19
    vpdpwssd        m5, m18, m14
    packuswb        m2, m4
    psrlw           m2, 8
    vpackuswb   m2{k2}, m3, m5
    movu    [dstq+r10], m2 ; We don't have a separate 5-tap version so the 7-tap
    add            r10, 64 ; function is used for chroma as well, and in some
    jl .hv_loop            ; esoteric edge cases chroma dst pointers may only
    mov             t6, t5 ; have a 32-byte alignment despite having a width
    mov             t5, t4 ; larger than 32, so use an unaligned store here.
    mov             t4, t3
    mov             t3, t2
    mov             t2, t1
    mov             t1, t0
    mov             t0, t6
    add           dstq, strideq
    ret
.v:
    mov            r10, wq
.v_loop:
    mova            m4, [t4+r10*2+ 0]
    paddw           m4, [t2+r10*2+ 0]
    mova            m1, [t3+r10*2+ 0]
    mova            m5, [t4+r10*2+64]
    paddw           m5, [t2+r10*2+64]
    mova            m3, [t3+r10*2+64]
    punpcklwd       m6, m4, m1
    mova            m0, m13
    vpdpwssd        m0, m6, m15
    punpcklwd       m6, m5, m3
    mova            m2, m13
    vpdpwssd        m2, m6, m15
    punpckhwd       m4, m1
    mova            m1, m13
    vpdpwssd        m1, m4, m15
    punpckhwd       m5, m3
    mova            m3, m13
    vpdpwssd        m3, m5, m15
    mova            m5, [t1+r10*2+ 0]
    paddw           m4, m5, [t6+r10*2+ 0]
    paddw           m5, [t5+r10*2+ 0]
    mova            m7, [t1+r10*2+64]
    paddw           m6, m7, [t6+r10*2+64]
    paddw           m7, [t5+r10*2+64]
    punpcklwd       m8, m4, m5
    vpdpwssd        m0, m8, m14
    punpcklwd       m8, m6, m7
    vpdpwssd        m2, m8, m14
    punpckhwd       m4, m5
    vpdpwssd        m1, m4, m14
    punpckhwd       m6, m7
    vpdpwssd        m3, m6, m14
    packuswb        m0, m2
    psrlw           m0, 8
    vpackuswb   m0{k2}, m1, m3
    movu    [dstq+r10], m0
    add            r10, 64
    jl .v_loop
    mov             t6, t5
    mov             t5, t4
    mov             t4, t3
    mov             t3, t2
    mov             t2, t1
    add           dstq, strideq
    ret
.w32:
    lea            r10, [r_ext_mask+73]
    mova          ym18, [wiener_perm32]
    lea             t1, [rsp+16]
    sub            r10, wq
    test         edgeb, 4 ; LR_HAVE_TOP
    jz .w32_no_top
    call .w32_h_top
    add           lpfq, strideq
    mov             t6, t1
    mov             t5, t1
    add             t1, 32*2
    call .w32_h_top
    lea             r9, [lpfq+strideq*4]
    mov           lpfq, dstq
    mov             t4, t1
    add             t1, 32*2
    add             r9, strideq
    mov          [rsp], r9 ; below
    call .w32_h
    mov             t3, t1
    mov             t2, t1
    dec             hd
    jz .w32_v1
    add           lpfq, strideq
    add             t1, 32*2
    call .w32_h
    mov             t2, t1
    dec             hd
    jz .w32_v2
    add           lpfq, strideq
    add             t1, 32*2
    call .w32_h
    dec             hd
    jz .w32_v3
.w32_main:
    lea             t0, [t1+32*2]
.w32_main_loop:
    call .w32_hv
    dec             hd
    jnz .w32_main_loop
    test         edgeb, 8 ; LR_HAVE_BOTTOM
    jz .w32_v3
    mov           lpfq, [rsp]
    call .w32_hv_bottom
    add           lpfq, strideq
    call .w32_hv_bottom
.w32_v1:
    call .w32_v
    RET
.w32_no_top:
    lea             r9, [lpfq+strideq*4]
    mov           lpfq, dstq
    lea             r9, [r9+strideq*2]
    mov          [rsp], r9
    call .w32_h
    mov             t6, t1
    mov             t5, t1
    mov             t4, t1
    mov             t3, t1
    mov             t2, t1
    dec             hd
    jz .w32_v1
    add           lpfq, strideq
    add             t1, 32*2
    call .w32_h
    mov             t2, t1
    dec             hd
    jz .w32_v2
    add           lpfq, strideq
    add             t1, 32*2
    call .w32_h
    dec             hd
    jz .w32_v3
    lea             t0, [t1+32*2]
    call .w32_hv
    dec             hd
    jz .w32_v3
    add             t0, 32*8
    call .w32_hv
    dec             hd
    jnz .w32_main
.w32_v3:
    call .w32_v
.w32_v2:
    call .w32_v
    jmp .w32_v1
.w32_h:
    test         edgeb, 1 ; LR_HAVE_LEFT
    jz .w32_h_extend_left
    movd          xm16, [leftq]
    vmovdqu32 ym16{k1}, [lpfq-4]
    add          leftq, 4
    jmp .w32_h_main
.w32_h_extend_left:
    vpbroadcastb  xm16, [lpfq]   ; the masked load ensures that no exception
    vmovdqu32 ym16{k1}, [lpfq-4] ; gets raised from accessing invalid memory
    jmp .w32_h_main
.w32_h_top:
    test         edgeb, 1 ; LR_HAVE_LEFT
    jz .w32_h_extend_left
    movu          ym16, [lpfq-4]
.w32_h_main:
    vinserti32x8   m16, [lpfq+4], 1
    test         edgeb, 2 ; LR_HAVE_RIGHT
    jnz .w32_h_have_right
    vpbroadcastb    m0, [lpfq+wq-1]
    movu          ym17, [r10-8]
    vinserti32x8   m17, [r10+0], 1
    vpternlogd     m16, m0, m17, 0xe4 ; c ? a : b
.w32_h_have_right:
    pshufb          m2, m16, m6
    mova            m0, m10
    vpdpbusd        m0, m2, m11
    pshufb          m2, m16, m7
    mova            m1, m10
    vpdpbusd        m1, m2, m11
    pshufb          m2, m16, m8
    vpdpbusd        m0, m2, m12
    pshufb         m16, m9
    vpdpbusd        m1, m16, m12
    packssdw        m0, m1
    psraw           m0, 3
    mova          [t1], m0
    ret
.w32_hv:
    add           lpfq, strideq
    test         edgeb, 1 ; LR_HAVE_LEFT
    jz .w32_hv_extend_left
    movd          xm16, [leftq]
    vmovdqu32 ym16{k1}, [lpfq-4]
    add          leftq, 4
    jmp .w32_hv_main
.w32_hv_extend_left:
    vpbroadcastb  xm16, [lpfq]
    vmovdqu32 ym16{k1}, [lpfq-4]
    jmp .w32_hv_main
.w32_hv_bottom:
    test         edgeb, 1 ; LR_HAVE_LEFT
    jz .w32_hv_extend_left
    movu          ym16, [lpfq-4]
.w32_hv_main:
    vinserti32x8   m16, [lpfq+4], 1
    test         edgeb, 2 ; LR_HAVE_RIGHT
    jnz .w32_hv_have_right
    vpbroadcastb    m0, [lpfq+wq-1]
    movu          ym17, [r10-8]
    vinserti32x8   m17, [r10+0], 1
    vpternlogd     m16, m0, m17, 0xe4
.w32_hv_have_right:
    mova            m3, [t4]
    paddw           m3, [t2]
    mova            m2, [t3]
    pshufb          m4, m16, m6
    mova            m0, m10
    vpdpbusd        m0, m4, m11
    pshufb          m4, m16, m7
    mova            m5, m10
    vpdpbusd        m5, m4, m11
    punpcklwd       m4, m3, m2
    mova            m1, m13
    vpdpwssd        m1, m4, m15
    punpckhwd       m3, m2
    mova            m2, m13
    vpdpwssd        m2, m3, m15
    pshufb          m4, m16, m8
    vpdpbusd        m0, m4, m12
    pshufb         m16, m9
    vpdpbusd        m5, m16, m12
    packssdw        m0, m5
    psraw           m0, 3
    mova            m4, [t5]
    paddw           m4, [t1]
    paddw           m3, m0, [t6]
    mova          [t0], m0
    punpcklwd       m0, m3, m4
    vpdpwssd        m1, m0, m14
    punpckhwd       m3, m4
    vpdpwssd        m2, m3, m14
    packuswb        m1, m2
    vpermb         m16, m18, m1
    mova        [dstq], ym16
    mov             t6, t5
    mov             t5, t4
    mov             t4, t3
    mov             t3, t2
    mov             t2, t1
    mov             t1, t0
    mov             t0, t6
    add           dstq, strideq
    ret
.w32_v:
    mova            m2, [t4]
    paddw           m2, [t2]
    mova            m1, [t3]
    mova            m4, [t1]
    paddw           m3, m4, [t6]
    paddw           m4, [t5]
    punpcklwd       m5, m2, m1
    mova            m0, m13
    vpdpwssd        m0, m5, m15
    punpckhwd       m2, m1
    mova            m1, m13
    vpdpwssd        m1, m2, m15
    punpcklwd       m2, m3, m4
    vpdpwssd        m0, m2, m14
    punpckhwd       m3, m4
    vpdpwssd        m1, m3, m14
    packuswb        m0, m1
    vpermb         m16, m18, m0
    mova        [dstq], ym16
    mov             t6, t5
    mov             t5, t4
    mov             t4, t3
    mov             t3, t2
    mov             t2, t1
    add           dstq, strideq
    ret

cglobal sgr_filter_5x5_8bpc, 4, 13, 23, 416*24+16, dst, stride, left, lpf, \
                                                   w, h, edge, params
    mov        paramsq, r6mp
    mov             wd, wm
    mov             hd, hm
    mov          edged, r7m
    vbroadcasti32x4 m5, [sgr_shuf+1]
    add           lpfq, wq
    vbroadcasti32x4 m6, [sgr_shuf+9]
    add           dstq, wq
    vbroadcasti32x4 m7, [sgr_shuf+3]
    lea             t3, [rsp+wq*4+16+416*12]
    vbroadcasti32x4 m8, [sgr_shuf+7]
    pxor            m4, m4
    vpbroadcastd    m9, [pd_m25]
    vpsubd         m11, m4, [paramsq+0] {1to16} ; -s0
    vpbroadcastw   m15, [paramsq+8]             ; w0
    lea             t1, [rsp+wq*2+20]
    vpbroadcastd   m10, [pw_164_455]
    neg             wq
    vpbroadcastd   m12, [pw_61448]              ; (15 << 12) + (1 << 3)
    mov           r10d, 0xfe
    vpbroadcastd   m13, [pd_m4096]
    kmovb           k1, r10d
    vpbroadcastd   m14, [pd_34816]              ; (1 << 11) + (1 << 15)
    mov            r10, 0x3333333333333333
    mova           m18, [sgr_x_by_x+64*0]
    kmovq           k2, r10
    mova           m19, [sgr_x_by_x+64*1]
    lea            r12, [r_ext_mask+75]
    mova           m20, [sgr_x_by_x+64*2]
    psllw          m15, 4
    mova           m21, [sgr_x_by_x+64*3]
    lea            r10, [lpfq+strideq*4]
    mova          ym22, [sgr_shuf]
    add            r10, strideq
    mov          [rsp], r10 ; below
    test         edgeb, 4 ; LR_HAVE_TOP
    jz .no_top
    call .h_top
    add           lpfq, strideq
    mov             t2, t1
    call .top_fixup
    add             t1, 416*6
    call .h_top
    lea            r10, [lpfq+strideq*4]
    mov           lpfq, dstq
    add            r10, strideq
    mov          [rsp], r10 ; below
    mov             t0, t2
    dec             hd
    jz .height1
    or           edged, 16
    call .h
.main:
    add           lpfq, strideq
    call .hv
    call .prep_n
    sub             hd, 2
    jl .extend_bottom
.main_loop:
    add           lpfq, strideq
    test            hd, hd
    jz .odd_height
    call .h
    add           lpfq, strideq
    call .hv
    call .n0
    call .n1
    sub             hd, 2
    jge .main_loop
    test         edgeb, 8 ; LR_HAVE_BOTTOM
    jz .extend_bottom
    mov           lpfq, [rsp]
    call .h_top
    add           lpfq, strideq
    call .hv_bottom
.end:
    call .n0
    call .n1
.end2:
    RET
.height1:
    call .hv
    call .prep_n
    jmp .odd_height_end
.odd_height:
    call .hv
    call .n0
    call .n1
.odd_height_end:
    call .v
    call .n0
    jmp .end2
.extend_bottom:
    call .v
    jmp .end
.no_top:
    lea            r10, [lpfq+strideq*4]
    mov           lpfq, dstq
    lea            r10, [r10+strideq*2]
    mov          [rsp], r10
    call .h
    lea             t2, [t1+416*6]
    call .top_fixup
    dec             hd
    jz .no_top_height1
    or           edged, 16
    mov             t0, t1
    mov             t1, t2
    jmp .main
.no_top_height1:
    call .v
    call .prep_n
    jmp .odd_height_end
.h: ; horizontal boxsum
    lea            r10, [wq-2]
    test         edgeb, 1 ; LR_HAVE_LEFT
    jz .h_extend_left
    movd          xm17, [leftq]
    vmovdqu32 ym17{k1}, [lpfq+wq-4]
    add          leftq, 4
    jmp .h_main
.h_extend_left:
    vpbroadcastb  xm17, [lpfq+wq]
    vmovdqu32 ym17{k1}, [lpfq+wq-4]
    jmp .h_main
.h_top:
    lea            r10, [wq-2]
    test         edgeb, 1 ; LR_HAVE_LEFT
    jz .h_extend_left
.h_loop:
    movu          ym17, [lpfq+r10-2]
.h_main:
    vinserti32x8   m17, [lpfq+r10+6], 1
    test         edgeb, 2 ; LR_HAVE_RIGHT
    jnz .h_have_right
    cmp           r10d, -34
    jl .h_have_right
    vpbroadcastb    m0, [lpfq-1]
    movu          ym16, [r12+r10-8]
    vinserti32x8   m16, [r12+r10+0], 1
    vpternlogd     m17, m0, m16, 0xe4
.h_have_right:
    pshufb          m3, m17, m5
    pmullw          m2, m3, m3
    pshufb          m1, m17, m6
    paddw           m0, m3, m1
    shufps          m3, m1, q2121
    paddw           m0, m3
    punpcklwd      m16, m3, m1
    punpckhwd       m3, m1
    punpcklwd       m1, m2, m4
    vpdpwssd        m1, m16, m16
    punpckhwd       m2, m4
    vpdpwssd        m2, m3, m3
    pshufb         m16, m17, m7
    paddw           m0, m16
    pshufb         m17, m8
    paddw           m0, m17              ; sum
    punpcklwd       m3, m16, m17
    vpdpwssd        m1, m3, m3           ; sumsq
    punpckhwd      m16, m17
    vpdpwssd        m2, m16, m16
    test         edgeb, 16 ; y > 0
    jz .h_loop_end
    paddw           m0, [t1+r10*2+416*0]
    paddd           m1, [t1+r10*2+416*2]
    paddd           m2, [t1+r10*2+416*4]
.h_loop_end:
    mova [t1+r10*2+416*0], m0
    mova [t1+r10*2+416*2], m1
    mova [t1+r10*2+416*4], m2
    add            r10, 32
    jl .h_loop
    ret
.top_fixup:
    lea            r10, [wq-2]
.top_fixup_loop: ; the sums of the first row needs to be doubled
    mova            m0, [t1+r10*2+416*0]
    mova            m1, [t1+r10*2+416*2]
    mova            m2, [t1+r10*2+416*4]
    paddw           m0, m0
    paddd           m1, m1
    paddd           m2, m2
    mova [t2+r10*2+416*0], m0
    mova [t2+r10*2+416*2], m1
    mova [t2+r10*2+416*4], m2
    add            r10, 32
    jl .top_fixup_loop
    ret
ALIGN function_align
.hv: ; horizontal boxsum + vertical boxsum + ab
    lea            r10, [wq-2]
    test         edgeb, 1 ; LR_HAVE_LEFT
    jz .hv_extend_left
    movd          xm17, [leftq]
    vmovdqu32 ym17{k1}, [lpfq+wq-4]
    add          leftq, 4
    jmp .hv_main
.hv_extend_left:
    vpbroadcastb  xm17, [lpfq+wq]
    vmovdqu32 ym17{k1}, [lpfq+wq-4]
    jmp .hv_main
.hv_bottom:
    lea            r10, [wq-2]
    test         edgeb, 1 ; LR_HAVE_LEFT
    jz .hv_extend_left
.hv_loop:
    movu          ym17, [lpfq+r10-2]
.hv_main:
    vinserti32x8   m17, [lpfq+r10+6], 1
    test         edgeb, 2 ; LR_HAVE_RIGHT
    jnz .hv_have_right
    cmp           r10d, -34
    jl .hv_have_right
    vpbroadcastb    m0, [lpfq-1]
    movu          ym16, [r12+r10-8]
    vinserti32x8   m16, [r12+r10+0], 1
    vpternlogd     m17, m0, m16, 0xe4
.hv_have_right:
    pshufb          m1, m17, m5
    pmullw          m3, m1, m1
    pshufb          m2, m17, m6
    paddw           m0, m1, m2
    shufps          m1, m2, q2121
    paddw           m0, m1
    punpcklwd      m16, m1, m2
    punpckhwd       m1, m2
    punpcklwd       m2, m3, m4
    vpdpwssd        m2, m16, m16
    punpckhwd       m3, m4
    vpdpwssd        m3, m1, m1
    pshufb         m16, m17, m7
    paddw           m0, m16
    pshufb         m17, m8
    paddw           m0, m17              ; h sum
    punpcklwd       m1, m16, m17
    vpdpwssd        m2, m1, m1           ; h sumsq
    punpckhwd      m16, m17
    vpdpwssd        m3, m16, m16
    paddw           m1, m0, [t1+r10*2+416*0]
    paddd          m16, m2, [t1+r10*2+416*2]
    paddd          m17, m3, [t1+r10*2+416*4]
    test            hd, hd
    jz .hv_last_row
.hv_main2:
    paddd          m16, [t2+r10*2+416*2] ; hv sumsq
    paddd          m17, [t2+r10*2+416*4]
    paddw           m1, [t2+r10*2+416*0] ; hv sum
    mova [t0+r10*2+416*2], m2
    mova [t0+r10*2+416*4], m3
    mova [t0+r10*2+416*0], m0
    pmulld         m16, m9               ; -a * 25
    pmulld         m17, m9
    punpcklwd       m0, m1, m4           ; b
    vpdpwssd       m16, m0, m0           ; -p
    punpckhwd       m1, m4
    vpdpwssd       m17, m1, m1
    pmaddwd         m0, m10              ; b * 164
    pmaddwd         m1, m10
    pmulld         m16, m11              ; p * s
    pmulld         m17, m11
    vpalignr   m17{k2}, m16, m16, 2
    mova           m16, m20
    paddusw        m17, m12
    psraw          m17, 4                ; min(z, 255) - 256
    vpermt2b       m16, m17, m21         ; sgr_x_by_x[128..255]
    vpmovb2m        k3, m17
    vpermi2b       m17, m18, m19         ; sgr_x_by_x[  0..127]
    vmovdqu8   m17{k3}, m16              ; x
    pandn          m16, m13, m17
    psrld          m17, 16
    pmulld          m0, m16
    pmulld          m1, m17
    paddd           m0, m14              ; x * b * 164 + (1 << 11) + (1 << 15)
    paddd           m1, m14
    vpternlogd     m16, m0, m13, 0xd8    ; a | (b << 12)
    vpternlogd     m17, m1, m13, 0xd8
    mova          [t3+r10*4+  8], m16    ; The neighbor calculations requires
    mova          [t3+r10*4+ 24], xm17   ; 13 bits for a and 21 bits for b.
    vextracti32x4 [t3+r10*4+ 56], m17, 2 ; Packing them allows for 12+20, but
    mova          [t3+r10*4+ 72], m17    ; that gets us most of the way.
    vextracti128  [t3+r10*4+ 72], ym16, 1
    vextracti32x4 [t3+r10*4+104], m16, 3
    add            r10, 32
    jl .hv_loop
    mov             t2, t1
    mov             t1, t0
    mov             t0, t2
    ret
.hv_last_row: ; esoteric edge case for odd heights
    mova [t1+r10*2+416*0], m1
    paddw              m1, m0
    mova [t1+r10*2+416*2], m16
    paddd             m16, m2
    mova [t1+r10*2+416*4], m17
    paddd             m17, m3
    jmp .hv_main2
.v: ; vertical boxsum + ab
    lea            r10, [wq-2]
.v_loop:
    mova            m2, [t1+r10*2+416*2]
    paddd          m16, m2, [t2+r10*2+416*2]
    mova            m3, [t1+r10*2+416*4]
    paddd          m17, m3, [t2+r10*2+416*4]
    paddd           m2, m2
    paddd           m3, m3
    paddd          m16, m2               ; hv sumsq
    paddd          m17, m3
    pmulld         m16, m9               ; -a * 25
    pmulld         m17, m9
    mova            m0, [t1+r10*2+416*0]
    paddw           m1, m0, [t2+r10*2+416*0]
    paddw           m0, m0
    paddw           m1, m0               ; hv sum
    punpcklwd       m0, m1, m4           ; b
    vpdpwssd       m16, m0, m0           ; -p
    punpckhwd       m1, m4
    vpdpwssd       m17, m1, m1
    pmaddwd         m0, m10              ; b * 164
    pmaddwd         m1, m10
    pmulld         m16, m11              ; p * s
    pmulld         m17, m11
    vpalignr   m17{k2}, m16, m16, 2
    mova           m16, m20
    paddusw        m17, m12
    psraw          m17, 4                ; min(z, 255) - 256
    vpermt2b       m16, m17, m21         ; sgr_x_by_x[128..255]
    vpmovb2m        k3, m17
    vpermi2b       m17, m18, m19         ; sgr_x_by_x[  0..127]
    vmovdqu8   m17{k3}, m16              ; x
    pandn          m16, m13, m17
    psrld          m17, 16
    pmulld          m0, m16
    pmulld          m1, m17
    paddd           m0, m14              ; x * b * 164 + (1 << 11) + (1 << 15)
    paddd           m1, m14
    vpternlogd     m16, m0, m13, 0xd8    ; a | (b << 12)
    vpternlogd     m17, m1, m13, 0xd8
    mova          [t3+r10*4+  8], m16
    mova          [t3+r10*4+ 24], xm17
    vextracti32x4 [t3+r10*4+ 56], m17, 2
    mova          [t3+r10*4+ 72], m17
    vextracti128  [t3+r10*4+ 72], ym16, 1
    vextracti32x4 [t3+r10*4+104], m16, 3
    add            r10, 32
    jl .v_loop
    ret
.prep_n: ; initial neighbor setup
    mov            r10, wq
.prep_n_loop:
    movu            m0, [t3+r10*4+ 4]
    movu            m1, [t3+r10*4+68]
    paddd           m2, m0, [t3+r10*4+ 0]
    paddd           m3, m1, [t3+r10*4+64]
    paddd           m2, [t3+r10*4+ 8]
    paddd           m3, [t3+r10*4+72]
    paddd           m0, m2
    pslld           m2, 2
    paddd           m1, m3
    pslld           m3, 2
    paddd           m2, m0                ; ab 565
    paddd           m3, m1
    pandn           m0, m13, m2           ; a
    psrld           m2, 12                ; b
    pandn           m1, m13, m3
    psrld           m3, 12
    mova [t3+r10*4+416*4+ 0], m0
    mova [t3+r10*4+416*8+ 0], m2
    mova [t3+r10*4+416*4+64], m1
    mova [t3+r10*4+416*8+64], m3
    add            r10, 32
    jl .prep_n_loop
    ret
ALIGN function_align
.n0: ; neighbor + output (even rows)
    mov            r10, wq
.n0_loop:
    movu           m16, [t3+r10*4+ 4]
    movu           m17, [t3+r10*4+68]
    paddd           m0, m16, [t3+r10*4+ 0]
    paddd           m1, m17, [t3+r10*4+64]
    paddd           m0, [t3+r10*4+ 8]
    paddd           m1, [t3+r10*4+72]
    paddd          m16, m0
    pslld           m0, 2
    paddd          m17, m1
    pslld           m1, 2
    paddd           m0, m16
    paddd           m1, m17
    pandn          m16, m13, m0
    psrld           m0, 12
    pandn          m17, m13, m1
    psrld           m1, 12
    paddd           m2, m16, [t3+r10*4+416*4+ 0] ; a
    paddd           m3, m17, [t3+r10*4+416*4+64]
    mova [t3+r10*4+416*4+ 0], m16
    mova [t3+r10*4+416*4+64], m17
    paddd          m16, m0, [t3+r10*4+416*8+ 0] ; b + (1 << 8)
    paddd          m17, m1, [t3+r10*4+416*8+64]
    mova [t3+r10*4+416*8+ 0], m0
    mova [t3+r10*4+416*8+64], m1
    pmovzxbd        m0, [dstq+r10+ 0]
    pmovzxbd        m1, [dstq+r10+16]
    pmaddwd         m2, m0                      ; a * src
    pmaddwd         m3, m1
    packssdw        m0, m1
    psubd          m16, m2                      ; b - a * src + (1 << 8)
    psubd          m17, m3
    psrad          m16, 9
    psrad          m17, 9
    packssdw       m16, m17
    pmulhrsw       m16, m15
    paddw          m16, m0
    packuswb       m16, m16
    vpermd         m16, m22, m16
    mova    [dstq+r10], ym16
    add            r10, 32
    jl .n0_loop
    add           dstq, strideq
    ret
ALIGN function_align
.n1: ; neighbor + output (odd rows)
    mov            r10, wq
.n1_loop:
    pmovzxbd        m0, [dstq+r10+ 0]
    pmovzxbd        m1, [dstq+r10+16]
    pmaddwd         m2, m0, [t3+r10*4+416*4+ 0] ; a * src
    pmaddwd         m3, m1, [t3+r10*4+416*4+64]
    mova           m16, [t3+r10*4+416*8+ 0]     ; b + (1 << 7)
    mova           m17, [t3+r10*4+416*8+64]
    packssdw        m0, m1
    psubd          m16, m2                      ; b - a * src + (1 << 7)
    psubd          m17, m3
    psrad          m16, 8
    psrad          m17, 8
    packssdw       m16, m17
    pmulhrsw       m16, m15
    paddw          m16, m0
    packuswb       m16, m16
    vpermd         m16, m22, m16
    mova    [dstq+r10], ym16
    add            r10, 32
    jl .n1_loop
    add           dstq, strideq
    ret

cglobal sgr_filter_3x3_8bpc, 4, 15, 22, -416*28-16, dst, stride, left, lpf, \
                                                    w, h, edge, params
    mov        paramsq, r6mp
    mov             wd, wm
    movifnidn       hd, hm
    mov          edged, r7m
    vbroadcasti32x4 m5, [sgr_shuf+3]
    add           lpfq, wq
    vbroadcasti32x4 m6, [sgr_shuf+5]
    add           dstq, wq
    vbroadcasti32x4 m7, [sgr_shuf+7]
    pxor            m4, m4
    vpbroadcastd    m8, [pd_m9]
    vpsubd         m11, m4, [paramsq+4] {1to16} ; -s1
    vpbroadcastw   m15, [paramsq+10]            ; w1
    lea             t1, [rsp+wq*2+20]
    vpbroadcastd   m10, [pw_164_455]
    lea             t3, [rsp+wq*4+16+416*12]
    vpbroadcastd   m12, [pw_61448]              ; (15 << 12) + (1 << 3)
    neg             wq
    vpbroadcastd   m13, [pd_m4096]
    mov           r10d, 0xfe
    vpbroadcastd   m14, [pd_34816]              ; (1 << 11) + (1 << 15)
    kmovb           k1, r10d
    mova           m18, [sgr_x_by_x+64*0]
    mov            r10, 0x3333333333333333
    mova           m19, [sgr_x_by_x+64*1]
    kmovq           k2, r10
    mova           m20, [sgr_x_by_x+64*2]
    psllw          m15, 4
    mova           m21, [sgr_x_by_x+64*3]
    lea            r14, [r_ext_mask+75]
    mova           ym9, [sgr_shuf]
    test         edgeb, 4 ; LR_HAVE_TOP
    jz .no_top
    call .h_top
    add           lpfq, strideq
    mov             t2, t1
    add             t1, 416*6
    call .h_top
    lea             t4, [lpfq+strideq*4]
    mov           lpfq, dstq
    add             t4, strideq
    mov          [rsp], t4 ; below
    mov             t0, t2
    call .hv
.main:
    mov             t5, t3
    add             t3, 416*4
    dec             hd
    jz .height1
    add           lpfq, strideq
    call .hv
    call .prep_n
    dec             hd
    jz .extend_bottom
.main_loop:
    add           lpfq, strideq
    call .hv
    call .n
    dec             hd
    jnz .main_loop
    test         edgeb, 8 ; LR_HAVE_BOTTOM
    jz .extend_bottom
    mov           lpfq, [rsp]
    call .hv_bottom
    call .n
    add           lpfq, strideq
    call .hv_bottom
.end:
    call .n
    RET
.height1:
    call .v
    call .prep_n
    mov             t2, t1
    call .v
    jmp .end
.extend_bottom:
    call .v
    call .n
    mov             t2, t1
    call .v
    jmp .end
.no_top:
    lea             t4, [lpfq+strideq*4]
    mov           lpfq, dstq
    lea             t4, [t4+strideq*2]
    mov          [rsp], t4
    call .h
    lea             t0, [t1+416*6]
    mov             t2, t1
    call .v
    jmp .main
.h: ; horizontal boxsum
    lea            r10, [wq-2]
    test         edgeb, 1 ; LR_HAVE_LEFT
    jz .h_extend_left
    movd          xm17, [leftq]
    vmovdqu32 ym17{k1}, [lpfq+wq-4]
    add          leftq, 4
    jmp .h_main
.h_extend_left:
    vpbroadcastb  xm17, [lpfq+wq]
    vmovdqu32 ym17{k1}, [lpfq+wq-4]
    jmp .h_main
.h_top:
    lea            r10, [wq-2]
    test         edgeb, 1 ; LR_HAVE_LEFT
    jz .h_extend_left
.h_loop:
    movu          ym17, [lpfq+r10-2]
.h_main:
    vinserti32x8   m17, [lpfq+r10+6], 1
    test         edgeb, 2 ; LR_HAVE_RIGHT
    jnz .h_have_right
    cmp           r10d, -33
    jl .h_have_right
    vpbroadcastb    m0, [lpfq-1]
    movu          ym16, [r14+r10-8]
    vinserti32x8   m16, [r14+r10+0], 1
    vpternlogd     m17, m0, m16, 0xe4
.h_have_right:
    pshufb          m0, m17, m5
    pmullw          m2, m0, m0
    pshufb         m16, m17, m6
    paddw           m0, m16
    pshufb         m17, m7
    paddw           m0, m17    ; sum
    punpcklwd       m3, m16, m17
    punpcklwd       m1, m2, m4
    vpdpwssd        m1, m3, m3 ; sumsq
    punpckhwd      m16, m17
    punpckhwd       m2, m4
    vpdpwssd        m2, m16, m16
    mova [t1+r10*2+416*0], m0
    mova [t1+r10*2+416*2], m1
    mova [t1+r10*2+416*4], m2
    add            r10, 32
    jl .h_loop
    ret
ALIGN function_align
.hv: ; horizontal boxsum + vertical boxsum + ab
    lea            r10, [wq-2]
    test         edgeb, 1 ; LR_HAVE_LEFT
    jz .hv_extend_left
    movd          xm17, [leftq]
    vmovdqu32 ym17{k1}, [lpfq+wq-4]
    add          leftq, 4
    jmp .hv_main
.hv_extend_left:
    vpbroadcastb  xm17, [lpfq+wq]
    vmovdqu32 ym17{k1}, [lpfq+wq-4]
    jmp .hv_main
.hv_bottom:
    lea            r10, [wq-2]
    test         edgeb, 1 ; LR_HAVE_LEFT
    jz .hv_extend_left
.hv_loop:
    movu          ym17, [lpfq+r10-2]
.hv_main:
    vinserti32x8   m17, [lpfq+r10+6], 1
    test         edgeb, 2 ; LR_HAVE_RIGHT
    jnz .hv_have_right
    cmp           r10d, -33
    jl .hv_have_right
    vpbroadcastb    m0, [lpfq-1]
    movu          ym16, [r14+r10-8]
    vinserti32x8   m16, [r14+r10+0], 1
    vpternlogd     m17, m0, m16, 0xe4
.hv_have_right:
    pshufb          m0, m17, m5
    pmullw          m3, m0, m0
    pshufb          m1, m17, m6
    paddw           m0, m1
    pshufb         m17, m7
    paddw           m0, m17              ; h sum
    punpcklwd      m16, m17, m1
    punpcklwd       m2, m3, m4
    vpdpwssd        m2, m16, m16         ; h sumsq
    punpckhwd      m17, m1
    punpckhwd       m3, m4
    vpdpwssd        m3, m17, m17
    paddw           m1, m0, [t2+r10*2+416*0]
    paddw           m1, [t1+r10*2+416*0] ; hv sum
    paddd          m16, m2, [t2+r10*2+416*2]
    paddd          m17, m3, [t2+r10*2+416*4]
    paddd          m16, [t1+r10*2+416*2] ; hv sumsq
    paddd          m17, [t1+r10*2+416*4]
    mova [t0+r10*2+416*0], m0
    mova [t0+r10*2+416*2], m2
    mova [t0+r10*2+416*4], m3
    pmulld         m16, m8               ; -a * 9
    pmulld         m17, m8
    punpcklwd       m0, m4, m1           ; b
    vpdpwssd       m16, m0, m0           ; -p
    punpckhwd       m1, m4, m1
    vpdpwssd       m17, m1, m1
    pmaddwd         m0, m10              ; b * 455
    pmaddwd         m1, m10
    pmulld         m16, m11              ; p * s
    pmulld         m17, m11
    vpalignr   m17{k2}, m16, m16, 2
    mova           m16, m20
    paddusw        m17, m12
    psraw          m17, 4                ; min(z, 255) - 256
    vpermt2b       m16, m17, m21         ; sgr_x_by_x[128..255]
    vpmovb2m        k3, m17
    vpermi2b       m17, m18, m19         ; sgr_x_by_x[  0..127]
    vmovdqu8   m17{k3}, m16              ; x
    pandn          m16, m13, m17
    psrld          m17, 16
    pmulld          m0, m16
    pmulld          m1, m17
    paddd           m0, m14              ; x * b * 455 + (1 << 11) + (1 << 15)
    paddd           m1, m14
    vpternlogd     m16, m0, m13, 0xd8    ; a | (b << 12)
    vpternlogd     m17, m1, m13, 0xd8
    mova          [t3+r10*4+  8], m16
    mova          [t3+r10*4+ 24], xm17
    vextracti32x4 [t3+r10*4+ 56], m17, 2
    mova          [t3+r10*4+ 72], m17
    vextracti128  [t3+r10*4+ 72], ym16, 1
    vextracti32x4 [t3+r10*4+104], m16, 3
    add            r10, 32
    jl .hv_loop
    mov             t2, t1
    mov             t1, t0
    mov             t0, t2
    ret
.v: ; vertical boxsum + ab
    lea            r10, [wq-2]
.v_loop:
    mova           m16, [t1+r10*2+416*2]
    mova           m17, [t1+r10*2+416*4]
    paddd          m16, m16
    paddd          m17, m17
    paddd          m16, [t2+r10*2+416*2] ; hv sumsq
    paddd          m17, [t2+r10*2+416*4]
    pmulld         m16, m8               ; -a * 9
    pmulld         m17, m8
    mova            m1, [t1+r10*2+416*0]
    paddw           m1, m1
    paddw           m1, [t2+r10*2+416*0] ; hv sum
    punpcklwd       m0, m4, m1           ; b
    vpdpwssd       m16, m0, m0           ; -p
    punpckhwd       m1, m4, m1
    vpdpwssd       m17, m1, m1
    pmaddwd         m0, m10              ; b * 455
    pmaddwd         m1, m10
    pmulld         m16, m11              ; p * s
    pmulld         m17, m11
    vpalignr   m17{k2}, m16, m16, 2
    mova           m16, m20
    paddusw        m17, m12
    psraw          m17, 4                ; min(z, 255) - 256
    vpermt2b       m16, m17, m21         ; sgr_x_by_x[128..255]
    vpmovb2m        k3, m17
    vpermi2b       m17, m18, m19         ; sgr_x_by_x[  0..127]
    vmovdqu8   m17{k3}, m16              ; x
    pandn          m16, m13, m17
    psrld          m17, 16
    pmulld          m0, m16
    pmulld          m1, m17
    paddd           m0, m14              ; x * b * 455 + (1 << 11) + (1 << 15)
    paddd           m1, m14
    vpternlogd     m16, m0, m13, 0xd8    ; a | (b << 12)
    vpternlogd     m17, m1, m13, 0xd8
    mova          [t3+r10*4+  8], m16
    mova          [t3+r10*4+ 24], xm17
    vextracti32x4 [t3+r10*4+ 56], m17, 2
    mova          [t3+r10*4+ 72], m17
    vextracti128  [t3+r10*4+ 72], ym16, 1
    vextracti32x4 [t3+r10*4+104], m16, 3
    add            r10, 32
    jl .v_loop
    ret
.prep_n: ; initial neighbor setup
    mov            r10, wq
    mov             t4, t3
    add             t3, 416*4
.prep_n_loop:
    mova            m2, [t5+r10*4+0]
    mova            m3, [t4+r10*4+0]
    paddd           m2, [t5+r10*4+8]
    paddd           m3, [t4+r10*4+8]
    paddd           m0, m2, [t5+r10*4+4]
    paddd           m1, m3, [t4+r10*4+4]
    pslld           m0, 2
    paddd           m1, m1                ; ab[ 0] 222
    psubd           m0, m2                ; ab[-1] 343
    mova [t3+r10*4+416*4], m1
    paddd           m1, m1
    mova    [t5+r10*4], m0
    psubd           m1, m3                ; ab[ 0] 343
    mova    [t4+r10*4], m1
    add            r10, 16
    jl .prep_n_loop
    ret
; a+b are packed together in a single dword, but we can't do the
; full neighbor calculations before splitting them since we don't
; have sufficient precision. The solution is to do the calculations
; in two equal halves and split a and b before doing the final sum.
ALIGN function_align
.n: ; neighbor + output
    mov            r10, wq
.n_loop:
    mova           m16, [t3+r10*4+ 0]
    paddd          m16, [t3+r10*4+ 8]
    paddd          m17, m16, [t3+r10*4+ 4]
    paddd          m17, m17               ; ab[+1] 222
    mova            m2, [t3+r10*4+416*4+ 0]
    paddd           m0, m2, [t5+r10*4+ 0] ; ab[ 0] 222 + ab[-1] 343
    mova            m3, [t3+r10*4+416*4+64]
    paddd           m1, m3, [t5+r10*4+64]
    mova [t3+r10*4+416*4+ 0], m17
    paddd          m17, m17
    psubd          m17, m16               ; ab[+1] 343
    mova [t5+r10*4+ 0], m17
    paddd           m2, m17               ; ab[ 0] 222 + ab[+1] 343
    mova           m16, [t3+r10*4+64]
    paddd          m16, [t3+r10*4+72]
    paddd          m17, m16, [t3+r10*4+68]
    paddd          m17, m17
    mova [t3+r10*4+416*4+64], m17
    paddd          m17, m17
    psubd          m17, m16
    mova [t5+r10*4+64], m17
    pandn          m16, m13, m0
    psrld           m0, 12
    paddd           m3, m17
    pandn          m17, m13, m2
    psrld           m2, 12
    paddd          m16, m17               ; a
    pandn          m17, m13, m1
    psrld           m1, 12
    paddd           m0, m2                ; b + (1 << 8)
    pandn           m2, m13, m3
    psrld           m3, 12
    paddd          m17, m2
    pmovzxbd        m2, [dstq+r10+ 0]
    paddd           m1, m3
    pmovzxbd        m3, [dstq+r10+16]
    pmaddwd        m16, m2                ; a * src
    pmaddwd        m17, m3
    packssdw        m2, m3
    psubd           m0, m16               ; b - a * src + (1 << 8)
    psubd           m1, m17
    psrad           m0, 9
    psrad           m1, 9
    packssdw        m0, m1
    pmulhrsw        m0, m15
    paddw           m0, m2
    packuswb        m0, m0
    vpermd         m16, m9, m0
    mova    [dstq+r10], ym16
    add            r10, 32
    jl .n_loop
    mov            r10, t5
    mov             t5, t4
    mov             t4, r10
    add           dstq, strideq
    ret

cglobal sgr_filter_mix_8bpc, 4, 13, 28, 416*56+8, dst, stride, left, lpf, \
                                                  w, h, edge, params
    mov        paramsq, r6mp
    mov             wd, wm
    movifnidn       hd, hm
    mov          edged, r7m
    vbroadcasti128  m5, [sgr_shuf+1]
    add           lpfq, wq
    vbroadcasti128  m6, [sgr_shuf+9]
    add           dstq, wq
    vbroadcasti128  m7, [sgr_shuf+3]
    lea             t3, [rsp+wq*4+416*24+8]
    vbroadcasti128  m8, [sgr_shuf+7]
    pxor            m4, m4
    vpbroadcastd    m9, [pd_m9]
    vpsubd         m11, m4, [paramsq+0] {1to16} ; -s0
    vpbroadcastd   m14, [pw_61448]
    vpsubd         m12, m4, [paramsq+4] {1to16} ; -s1
    vpbroadcastd   m26, [paramsq+8]             ; w0 w1
    lea             t1, [rsp+wq*2+12]
    vpbroadcastd   m10, [pd_m25]
    neg             wq
    vpbroadcastd   m13, [pw_164_455]
    mov           r10d, 0xfe
    vpbroadcastd   m15, [pd_34816]
    kmovb           k1, r10d
    mova           m20, [sgr_x_by_x+64*0]
    mov            r10, 0x3333333333333333
    mova           m21, [sgr_x_by_x+64*1]
    kmovq           k2, r10
    mova           m22, [sgr_x_by_x+64*2]
    lea            r12, [r_ext_mask+75]
    mova           m23, [sgr_x_by_x+64*3]
    vpbroadcastd   m24, [pd_m4096]
    vpbroadcastd   m25, [sgr_shuf+28]           ; 0x8000____
    psllw          m26, 5
    mova          xm27, [sgr_mix_perm]
    test         edgeb, 4 ; LR_HAVE_TOP
    jz .no_top
    call .h_top
    add           lpfq, strideq
    mov             t2, t1
    call mangle(private_prefix %+ _sgr_filter_5x5_8bpc_avx512icl).top_fixup
    add             t1, 416*12
    call .h_top
    lea            r10, [lpfq+strideq*4]
    mov           lpfq, dstq
    add            r10, strideq
    mov          [rsp], r10 ; below
    call .hv0
.main:
    dec             hd
    jz .height1
    add           lpfq, strideq
    call .hv1
    call .prep_n
    sub             hd, 2
    jl .extend_bottom
.main_loop:
    add           lpfq, strideq
    call .hv0
    test            hd, hd
    jz .odd_height
    add           lpfq, strideq
    call .hv1
    call .n0
    call .n1
    sub             hd, 2
    jge .main_loop
    test         edgeb, 8 ; LR_HAVE_BOTTOM
    jz .extend_bottom
    mov           lpfq, [rsp]
    call .hv0_bottom
    add           lpfq, strideq
    call .hv1_bottom
.end:
    call .n0
    call .n1
.end2:
    RET
.height1:
    call .v1
    call .prep_n
    jmp .odd_height_end
.odd_height:
    call .v1
    call .n0
    call .n1
.odd_height_end:
    call .v0
    call .v1
    call .n0
    jmp .end2
.extend_bottom:
    call .v0
    call .v1
    jmp .end
.no_top:
    lea            r10, [lpfq+strideq*4]
    mov           lpfq, dstq
    lea            r10, [r10+strideq*2]
    mov          [rsp], r10
    call .h
    lea             t2, [t1+416*12]
    lea            r10, [wq-2]
.top_fixup_loop:
    mova            m0, [t1+r10*2+416* 0]
    mova            m1, [t1+r10*2+416* 2]
    mova            m2, [t1+r10*2+416* 4]
    paddw           m0, m0
    mova            m3, [t1+r10*2+416* 6]
    paddd           m1, m1
    mova           m16, [t1+r10*2+416* 8]
    paddd           m2, m2
    mova           m17, [t1+r10*2+416*10]
    mova [t2+r10*2+416* 0], m0
    mova [t2+r10*2+416* 2], m1
    mova [t2+r10*2+416* 4], m2
    mova [t2+r10*2+416* 6], m3
    mova [t2+r10*2+416* 8], m16
    mova [t2+r10*2+416*10], m17
    add            r10, 32
    jl .top_fixup_loop
    call .v0
    jmp .main
.h: ; horizontal boxsums
    lea            r10, [wq-2]
    test         edgeb, 1 ; LR_HAVE_LEFT
    jz .h_extend_left
    movd          xm17, [leftq]
    vmovdqu32 ym17{k1}, [lpfq+wq-4]
    add          leftq, 4
    jmp .h_main
.h_extend_left:
    vpbroadcastb  xm17, [lpfq+wq]
    vmovdqu32 ym17{k1}, [lpfq+wq-4]
    jmp .h_main
.h_top:
    lea            r10, [wq-2]
    test         edgeb, 1 ; LR_HAVE_LEFT
    jz .h_extend_left
.h_loop:
    movu          ym17, [lpfq+r10-2]
.h_main:
    vinserti32x8   m17, [lpfq+r10+6], 1
    test         edgeb, 2 ; LR_HAVE_RIGHT
    jnz .h_have_right
    cmp           r10d, -34
    jl .h_have_right
    vpbroadcastb    m0, [lpfq-1]
    movu          ym16, [r12+r10-8]
    vinserti32x8   m16, [r12+r10+0], 1
    vpternlogd     m17, m0, m16, 0xe4
.h_have_right:
    pshufb          m3, m17, m5
    pshufb         m18, m17, m6
    shufps          m0, m3, m18, q2121
    pmullw          m2, m0, m0
    pshufb         m19, m17, m7
    paddw           m0, m19
    pshufb         m17, m8
    paddw           m0, m17           ; sum3
    punpcklwd      m16, m19, m17
    punpcklwd       m1, m2, m4
    vpdpwssd        m1, m16, m16      ; sumsq3
    punpckhwd      m19, m17
    punpckhwd       m2, m4
    vpdpwssd        m2, m19, m19
    mova [t1+r10*2+416* 6], m0
    mova [t1+r10*2+416* 8], m1
    mova [t1+r10*2+416*10], m2
    punpcklwd      m19, m3, m18
    paddw           m0, m3
    vpdpwssd        m1, m19, m19      ; sumsq5
    punpckhwd       m3, m18
    paddw           m0, m18           ; sum5
    vpdpwssd        m2, m3, m3
    mova [t1+r10*2+416* 0], m0
    mova [t1+r10*2+416* 2], m1
    mova [t1+r10*2+416* 4], m2
    add            r10, 32
    jl .h_loop
    ret
ALIGN function_align
.hv0: ; horizontal boxsums + vertical boxsum3 + ab3 (even rows)
    lea            r10, [wq-2]
    test         edgeb, 1 ; LR_HAVE_LEFT
    jz .hv0_extend_left
    movd          xm17, [leftq]
    vmovdqu32 ym17{k1}, [lpfq+wq-4]
    add          leftq, 4
    jmp .hv0_main
.hv0_extend_left:
    vpbroadcastb  xm17, [lpfq+wq]
    vmovdqu32 ym17{k1}, [lpfq+wq-4]
    jmp .hv0_main
.hv0_bottom:
    lea            r10, [wq-2]
    test         edgeb, 1 ; LR_HAVE_LEFT
    jz .hv0_extend_left
.hv0_loop:
    movu          ym17, [lpfq+r10-2]
.hv0_main:
    vinserti32x8   m17, [lpfq+r10+6], 1
    test         edgeb, 2 ; LR_HAVE_RIGHT
    jnz .hv0_have_right
    cmp           r10d, -34
    jl .hv0_have_right
    vpbroadcastb    m0, [lpfq-1]
    movu          ym16, [r12+r10-8]
    vinserti32x8   m16, [r12+r10+0], 1
    vpternlogd     m17, m0, m16, 0xe4
.hv0_have_right:
    pshufb         m18, m17, m5
    pshufb         m19, m17, m6
    shufps          m1, m18, m19, q2121
    pmullw          m3, m1, m1
    pshufb          m0, m17, m7
    paddw           m1, m0
    pshufb         m17, m8
    paddw           m1, m17           ; sum3
    punpcklwd      m16, m0, m17
    punpcklwd       m2, m3, m4
    vpdpwssd        m2, m16, m16      ; sumsq3
    punpckhwd       m0, m17
    punpckhwd       m3, m4
    vpdpwssd        m3, m0, m0
    paddw           m0, m1, [t1+r10*2+416* 6]
    paddd          m16, m2, [t1+r10*2+416* 8]
    paddd          m17, m3, [t1+r10*2+416*10]
    mova [t1+r10*2+416* 6], m1
    mova [t1+r10*2+416* 8], m2
    mova [t1+r10*2+416*10], m3
    paddw           m1, m18
    paddw           m1, m19           ; sum5
    mova [t3+r10*4+416*8+ 8], m1
    paddw           m1, [t1+r10*2+416* 0]
    mova [t1+r10*2+416* 0], m1
    punpcklwd       m1, m18, m19
    vpdpwssd        m2, m1, m1        ; sumsq5
    punpckhwd      m18, m19
    vpdpwssd        m3, m18, m18
    mova [t3+r10*4+416*0+ 8], m2      ; we need a clean copy of the last row
    mova [t3+r10*4+416*0+72], m3      ; in case height is odd
    paddd           m2, [t1+r10*2+416* 2]
    paddd           m3, [t1+r10*2+416* 4]
    mova [t1+r10*2+416* 2], m2
    mova [t1+r10*2+416* 4], m3
    paddw           m1, m0, [t2+r10*2+416* 6]
    paddd           m2, m16, [t2+r10*2+416* 8]
    paddd           m3, m17, [t2+r10*2+416*10]
    mova [t2+r10*2+416* 6], m0
    mova [t2+r10*2+416* 8], m16
    mova [t2+r10*2+416*10], m17
    pmulld         m16, m2, m9        ; -a3 * 9
    pmulld         m17, m3, m9
    punpcklwd       m0, m4, m1        ; b3
    vpdpwssd       m16, m0, m0        ; -p3
    punpckhwd       m1, m4, m1
    vpdpwssd       m17, m1, m1
    pmulld         m16, m12           ; p3 * s1
    pmulld         m17, m12
    pmaddwd         m0, m13           ; b3 * 455
    pmaddwd         m1, m13
    vpalignr   m17{k2}, m16, m16, 2
    mova           m16, m22
    paddusw        m17, m14
    psraw          m17, 4             ; min(z3, 255) - 256
    vpermt2b       m16, m17, m23      ; sgr_x_by_x[128..255]
    vpmovb2m        k3, m17
    vpermi2b       m17, m20, m21      ; sgr_x_by_x[  0..127]
    vmovdqu8   m17{k3}, m16           ; x3
    pandn          m16, m24, m17
    psrld          m17, 16
    pmulld          m0, m16
    pmulld          m1, m17
    paddd           m0, m15           ; x3 * b3 * 455 + (1 << 11) + (1 << 15)
    paddd           m1, m15
    vpternlogd     m16, m0, m24, 0xd8 ; a3 | (b3 << 12)
    vpternlogd     m17, m1, m24, 0xd8
    mova          [t3+r10*4+416*4+  8], m16
    mova          [t3+r10*4+416*4+ 24], xm17
    vextracti32x4 [t3+r10*4+416*4+ 56], m17, 2
    mova          [t3+r10*4+416*4+ 72], m17
    vextracti128  [t3+r10*4+416*4+ 72], ym16, 1
    vextracti32x4 [t3+r10*4+416*4+104], m16, 3
    add            r10, 32
    jl .hv0_loop
    ret
ALIGN function_align
.hv1: ; horizontal boxsums + vertical boxsums + ab (odd rows)
    lea            r10, [wq-2]
    test         edgeb, 1 ; LR_HAVE_LEFT
    jz .hv1_extend_left
    movd          xm17, [leftq]
    vmovdqu32 ym17{k1}, [lpfq+wq-4]
    add          leftq, 4
    jmp .hv1_main
.hv1_extend_left:
    vpbroadcastb  xm17, [lpfq+wq]
    vmovdqu32 ym17{k1}, [lpfq+wq-4]
    jmp .hv1_main
.hv1_bottom:
    lea            r10, [wq-2]
    test         edgeb, 1 ; LR_HAVE_LEFT
    jz .hv1_extend_left
.hv1_loop:
    movu          ym17, [lpfq+r10-2]
.hv1_main:
    vinserti32x8   m17, [lpfq+r10+6], 1
    test         edgeb, 2 ; LR_HAVE_RIGHT
    jnz .hv1_have_right
    cmp           r10d, -34
    jl .hv1_have_right
    vpbroadcastb    m0, [lpfq-1]
    movu          ym16, [r12+r10-8]
    vinserti32x8   m16, [r12+r10+0], 1
    vpternlogd    m17, m0, m16, 0xe4
.hv1_have_right:
    pshufb          m3, m17, m5
    pshufb         m19, m17, m6
    shufps          m2, m3, m19, q2121
    pmullw          m1, m2, m2
    pshufb         m18, m17, m7
    paddw           m2, m18
    pshufb         m17, m8
    paddw           m2, m17           ; sum3
    punpcklwd      m16, m17, m18
    punpcklwd       m0, m1, m4
    vpdpwssd        m0, m16, m16      ; sumsq3
    punpckhwd      m17, m18
    punpckhwd       m1, m4
    vpdpwssd        m1, m17, m17
    paddd          m16, m0, [t2+r10*2+416* 8]
    paddd          m17, m1, [t2+r10*2+416*10]
    mova [t2+r10*2+416* 8], m0
    mova [t2+r10*2+416*10], m1
    punpcklwd      m18, m3, m19
    vpdpwssd        m0, m18, m18      ; sumsq5
    punpckhwd      m18, m3, m19
    vpdpwssd        m1, m18, m18
    paddw           m3, m19
    pmulld         m16, m9            ; -a3 * 9
    pmulld         m17, m9
    paddd          m18, m0, [t2+r10*2+416*2]
    paddd          m19, m1, [t2+r10*2+416*4]
    paddd          m18, [t1+r10*2+416*2]
    paddd          m19, [t1+r10*2+416*4]
    mova [t2+r10*2+416*2], m0
    mova [t2+r10*2+416*4], m1
    pmulld         m18, m10           ; -a5 * 25
    pmulld         m19, m10
    paddw           m1, m2, [t2+r10*2+416* 6]
    mova [t2+r10*2+416* 6], m2
    paddw           m2, m3            ; sum5
    paddw           m3, m2, [t2+r10*2+416*0]
    paddw           m3, [t1+r10*2+416*0]
    mova [t2+r10*2+416*0], m2
    punpcklwd       m0, m4, m1        ; b3
    vpdpwssd       m16, m0, m0        ; -p3
    punpckhwd       m1, m4, m1
    vpdpwssd       m17, m1, m1
    punpcklwd       m2, m3, m4        ; b5
    vpdpwssd       m18, m2, m2        ; -p5
    punpckhwd       m3, m4
    vpdpwssd       m19, m3, m3
    pmulld         m16, m12           ; p3 * s1
    pmulld         m17, m12
    pmulld         m18, m11           ; p5 * s0
    pmulld         m19, m11
    pmaddwd         m0, m13           ; b3 * 455
    pmaddwd         m1, m13
    pmaddwd         m2, m13           ; b5 * 164
    pmaddwd         m3, m13
    vpalignr   m17{k2}, m16, m16, 2
    vpalignr   m19{k2}, m18, m18, 2
    paddusw        m17, m14
    mova           m16, m22
    psraw          m17, 4             ; min(z3, 255) - 256
    vpermt2b       m16, m17, m23      ; sgr_x_by_x[128..255]
    vpmovb2m        k3, m17
    vpermi2b       m17, m20, m21      ; sgr_x_by_x[  0..127]
    paddusw        m19, m14
    mova           m18, m22
    psraw          m19, 4             ; min(z5, 255) - 256
    vpermt2b       m18, m19, m23      ; sgr_x_by_x[128..255]
    vpmovb2m        k4, m19
    vpermi2b       m19, m20, m21      ; sgr_x_by_x[  0..127]
    vmovdqu8   m17{k3}, m16           ; x3
    vmovdqu8   m19{k4}, m18           ; x5
    pandn          m16, m24, m17
    psrld          m17, 16
    pmulld          m0, m16
    pmulld          m1, m17
    pandn          m18, m24, m19
    psrld          m19, 16
    pmulld          m2, m18
    pmulld          m3, m19
    paddd           m0, m15           ; x3 * b3 * 455 + (1 << 11) + (1 << 15)
    paddd           m1, m15
    vpternlogd     m16, m0, m24, 0xd8 ; a3 | (b3 << 12)
    vpternlogd     m17, m1, m24, 0xd8
    mova          [t3+r10*4+416*8+  8], m16
    mova          [t3+r10*4+416*8+ 24], xm17
    vextracti32x4 [t3+r10*4+416*8+ 56], m17, 2
    paddd           m2, m15           ; x5 * b5 * 164 + (1 << 11) + (1 << 15)
    paddd           m3, m15
    mova          [t3+r10*4+416*8+ 72], m17
    vextracti128  [t3+r10*4+416*8+ 72], ym16, 1
    vextracti32x4 [t3+r10*4+416*8+104], m16, 3
    vpternlogd     m18, m2, m24, 0xd8 ; a5 | (b5 << 12)
    vpternlogd     m19, m3, m24, 0xd8
    mova          [t3+r10*4+416*0+  8], m18
    mova          [t3+r10*4+416*0+ 24], xm19
    vextracti32x4 [t3+r10*4+416*0+ 56], m19, 2
    mova          [t3+r10*4+416*0+ 72], m19
    vextracti128  [t3+r10*4+416*0+ 72], ym18, 1
    vextracti32x4 [t3+r10*4+416*0+104], m18, 3
    add            r10, 32
    jl .hv1_loop
    mov            r10, t2
    mov             t2, t1
    mov             t1, r10
    ret
.v0: ; vertical boxsums + ab3 (even rows)
    lea            r10, [wq-2]
.v0_loop:
    mova            m2, [t1+r10*2+416* 8]
    mova            m3, [t1+r10*2+416*10]
    paddd           m2, m2
    paddd           m3, m3
    paddd          m16, m2, [t2+r10*2+416* 8]
    paddd          m17, m3, [t2+r10*2+416*10]
    mova            m0, [t1+r10*2+416* 6]
    paddw           m0, m0
    paddw           m1, m0, [t2+r10*2+416* 6]
    pmulld         m16, m9            ; -a3 * 9
    pmulld         m17, m9
    mova [t2+r10*2+416* 6], m0
    mova [t2+r10*2+416* 8], m2
    mova [t2+r10*2+416*10], m3
    mova            m2, [t1+r10*2+416*0]
    mova            m3, [t1+r10*2+416*2]
    mova           m18, [t1+r10*2+416*4]
    punpcklwd       m0, m4, m1        ; b3
    vpdpwssd       m16, m0, m0        ; -p3
    punpckhwd       m1, m4, m1
    vpdpwssd       m17, m1, m1
    pmulld         m16, m12           ; p3 * s1
    pmulld         m17, m12
    pmaddwd         m0, m13           ; b3 * 455
    pmaddwd         m1, m13
    mova [t3+r10*4+416*8+ 8], m2
    mova [t3+r10*4+416*0+ 8], m3
    mova [t3+r10*4+416*0+72], m18
    vpalignr   m17{k2}, m16, m16, 2
    mova           m16, m22
    paddusw        m17, m14
    psraw          m17, 4             ; min(z3, 255) - 256
    vpermt2b       m16, m17, m23      ; sgr_x_by_x[128..255]
    vpmovb2m        k3, m17
    vpermi2b       m17, m20, m21      ; sgr_x_by_x[  0..127]
    vmovdqu8   m17{k3}, m16           ; x3
    pandn          m16, m24, m17
    psrld          m17, 16
    pmulld          m0, m16
    pmulld          m1, m17
    paddw           m2, m2            ; cc5
    paddd           m3, m3
    paddd          m18, m18
    mova [t1+r10*2+416*0], m2
    mova [t1+r10*2+416*2], m3
    mova [t1+r10*2+416*4], m18
    paddd           m0, m15           ; x3 * b3 * 455 + (1 << 11) + (1 << 15)
    paddd           m1, m15
    vpternlogd     m16, m0, m24, 0xd8 ; a3 | (b3 << 12)
    vpternlogd     m17, m1, m24, 0xd8
    mova          [t3+r10*4+416*4+  8], m16
    mova          [t3+r10*4+416*4+ 24], xm17
    vextracti32x4 [t3+r10*4+416*4+ 56], m17, 2
    mova          [t3+r10*4+416*4+ 72], m17
    vextracti128  [t3+r10*4+416*4+ 72], ym16, 1
    vextracti32x4 [t3+r10*4+416*4+104], m16, 3
    add            r10, 32
    jl .v0_loop
    ret
.v1: ; vertical boxsums + ab (odd rows)
    lea            r10, [wq-2]
.v1_loop:
    mova            m0, [t1+r10*2+416* 8]
    paddd          m16, m0, [t2+r10*2+416* 8]
    mova            m1, [t1+r10*2+416*10]
    paddd          m17, m1, [t2+r10*2+416*10]
    mova            m2, [t3+r10*4+416*0+ 8]
    paddd          m18, m2, [t2+r10*2+416* 2]
    mova            m3, [t3+r10*4+416*0+72]
    paddd          m19, m3, [t2+r10*2+416* 4]
    paddd          m18, [t1+r10*2+416* 2]
    paddd          m19, [t1+r10*2+416* 4]
    mova [t2+r10*2+416* 8], m0
    mova [t2+r10*2+416*10], m1
    mova [t2+r10*2+416* 2], m2
    mova [t2+r10*2+416* 4], m3
    pmulld         m16, m9            ; -a3 * 9
    pmulld         m17, m9
    pmulld         m18, m10           ; -a5 * 25
    pmulld         m19, m10
    mova            m0, [t1+r10*2+416* 6]
    paddw           m1, m0, [t2+r10*2+416* 6]
    mova            m2, [t3+r10*4+416*8+ 8]
    paddw           m3, m2, [t2+r10*2+416*0]
    paddw           m3, [t1+r10*2+416*0]
    mova [t2+r10*2+416* 6], m0
    mova [t2+r10*2+416*0], m2
    punpcklwd       m0, m4, m1        ; b3
    vpdpwssd       m16, m0, m0        ; -p3
    punpckhwd       m1, m4, m1
    vpdpwssd       m17, m1, m1
    punpcklwd       m2, m3, m4        ; b5
    vpdpwssd       m18, m2, m2        ; -p5
    punpckhwd       m3, m4
    vpdpwssd       m19, m3, m3
    pmulld         m16, m12           ; p3 * s1
    pmulld         m17, m12
    pmulld         m18, m11           ; p5 * s0
    pmulld         m19, m11
    pmaddwd         m0, m13           ; b3 * 455
    pmaddwd         m1, m13
    pmaddwd         m2, m13           ; b5 * 164
    pmaddwd         m3, m13
    vpalignr   m17{k2}, m16, m16, 2
    vpalignr   m19{k2}, m18, m18, 2
    paddusw        m17, m14
    mova           m16, m22
    psraw          m17, 4             ; min(z3, 255) - 256
    vpermt2b       m16, m17, m23      ; sgr_x_by_x[128..255]
    vpmovb2m        k3, m17
    vpermi2b       m17, m20, m21      ; sgr_x_by_x[  0..127]
    paddusw        m19, m14
    mova           m18, m22
    psraw          m19, 4             ; min(z5, 255) - 256
    vpermt2b       m18, m19, m23      ; sgr_x_by_x[128..255]
    vpmovb2m        k4, m19
    vpermi2b       m19, m20, m21      ; sgr_x_by_x[  0..127]
    vmovdqu8   m17{k3}, m16           ; x3
    vmovdqu8   m19{k4}, m18           ; x5
    pandn          m16, m24, m17
    psrld          m17, 16
    pmulld          m0, m16
    pmulld          m1, m17
    pandn          m18, m24, m19
    psrld          m19, m19, 16
    pmulld          m2, m18
    pmulld          m3, m19
    paddd           m0, m15           ; x3 * b3 * 455 + (1 << 11) + (1 << 15)
    paddd           m1, m15
    vpternlogd     m16, m0, m24, 0xd8 ; a3 | (b3 << 12)
    vpternlogd     m17, m1, m24, 0xd8
    mova          [t3+r10*4+416*8+  8], m16
    mova          [t3+r10*4+416*8+ 24], xm17
    vextracti32x4 [t3+r10*4+416*8+ 56], m17, 2
    paddd           m2, m15           ; x5 * b5 * 164 + (1 << 11) + (1 << 15)
    paddd           m3, m15
    mova          [t3+r10*4+416*8+ 72], m17
    vextracti128  [t3+r10*4+416*8+ 72], ym16, 1
    vextracti32x4 [t3+r10*4+416*8+104], m16, 3
    vpternlogd     m18, m2, m24, 0xd8 ; a5 | (b5 << 12)
    vpternlogd     m19, m3, m24, 0xd8
    mova          [t3+r10*4+416*0+  8], m18
    mova          [t3+r10*4+416*0+ 24], xm19
    vextracti32x4 [t3+r10*4+416*0+ 56], m19, 2
    mova          [t3+r10*4+416*0+ 72], m19
    vextracti128  [t3+r10*4+416*0+ 72], ym18, 1
    vextracti32x4 [t3+r10*4+416*0+104], m18, 3
    add            r10, 32
    jl .v1_loop
    mov            r10, t2
    mov             t2, t1
    mov             t1, r10
    ret
.prep_n: ; initial neighbor setup
    mov            r10, wq
.prep_n_loop:
    movu            m0, [t3+r10*4+416*0+4]
    paddd           m1, m0, [t3+r10*4+416*0+0]
    mova           m16, [t3+r10*4+416*4+0]
    paddd           m1, [t3+r10*4+416*0+8]
    mova           m17, [t3+r10*4+416*8+0]
    paddd          m16, [t3+r10*4+416*4+8]
    paddd          m17, [t3+r10*4+416*8+8]
    paddd           m2, m16, [t3+r10*4+416*4+4]
    paddd           m3, m17, [t3+r10*4+416*8+4]
    paddd           m0, m1
    pslld           m1, 2
    pslld           m2, 2
    paddd           m1, m0            ; ab5 565
    paddd           m3, m3            ; ab3[ 0] 222
    psubd           m2, m16           ; ab3[-1] 343
    mova [t3+r10*4+416*20], m3
    pandn           m0, m24, m1       ; a5 565
    mova [t3+r10*4+416*24], m2
    psrld           m1, 12            ; b5 565
    mova [t3+r10*4+416*12], m0
    paddd           m3, m3
    mova [t3+r10*4+416*16], m1
    psubd           m3, m17           ; ab3[ 0] 343
    mova [t3+r10*4+416*28], m3
    add            r10, 16
    jl .prep_n_loop
    ret
ALIGN function_align
.n0: ; neighbor + output (even rows)
    mov            r10, wq
.n0_loop:
    movu            m2, [t3+r10*4+4]
    paddd           m3, m2, [t3+r10*4+0]
    paddd           m3, [t3+r10*4+8]
    mova            m1, [t3+r10*4+416*4+0]
    paddd           m2, m3
    pslld           m3, 2
    paddd           m1, [t3+r10*4+416*4+8]
    paddd           m3, m2
    pandn           m2, m24, m3
    psrld           m3, 12
    paddd           m0, m2, [t3+r10*4+416*12] ; a5
    paddd          m16, m3, [t3+r10*4+416*16] ; b5 + (1 << 8)
    mova [t3+r10*4+416*12], m2
    mova [t3+r10*4+416*16], m3
    paddd           m2, m1, [t3+r10*4+416*4+4]
    paddd           m2, m2                    ; ab3[ 1] 222
    mova            m3, [t3+r10*4+416*20]
    paddd          m17, m3, [t3+r10*4+416*24] ; ab3[ 0] 222 + ab3[-1] 343
    mova [t3+r10*4+416*20], m2
    paddd           m2, m2
    psubd           m2, m1                    ; ab3[ 1] 343
    mova [t3+r10*4+416*24], m2
    paddd           m2, m3                    ; ab3[ 0] 222 + ab3[ 1] 343
    pandn           m1, m24, m17
    psrld          m17, 12
    pandn           m3, m24, m2
    psrld           m2, 12
    paddd           m1, m3                    ; a3
    pmovzxbd        m3, [dstq+r10]
    paddd          m17, m2                    ; b3 + (1 << 8)
    pmaddwd         m0, m3                    ; a5 * src
    pmaddwd         m1, m3                    ; a3 * src
    vpshldd         m3, m25, 16               ; (dst << 16) + (1 << 15)
    psubd          m16, m0                    ; b5 - a5 * src + (1 << 8)
    psubd          m17, m1                    ; b3 - a3 * src + (1 << 8)
    psrld          m16, 9
    pslld          m17, 7
    vmovdqu8   m17{k2}, m16
    vpdpwssd        m3, m17, m26
    packuswb        m3, m2
    vpermb         m16, m27, m3
    mova    [dstq+r10], xm16
    add            r10, 16
    jl .n0_loop
    add           dstq, strideq
    ret
ALIGN function_align
.n1: ; neighbor + output (odd rows)
    mov            r10, wq
.n1_loop:
    mova            m1, [t3+r10*4+416*8+0]
    paddd           m1, [t3+r10*4+416*8+8]
    paddd           m2, m1, [t3+r10*4+416*8+4]
    paddd           m2, m2                    ; ab3[ 1] 222
    mova            m0, [t3+r10*4+416*20]
    paddd          m17, m0, [t3+r10*4+416*28] ; ab3[ 0] 222 + ab3[-1] 343
    pmovzxbd        m3, [dstq+r10]
    mova [t3+r10*4+416*20], m2
    paddd           m2, m2
    psubd           m2, m1                    ; ab3[ 1] 343
    mova [t3+r10*4+416*28], m2
    paddd           m0, m2                    ; ab3[ 0] 222 + ab3[ 1] 343
    pandn           m1, m24, m17
    psrld          m17, 12
    pandn           m2, m24, m0
    psrld           m0, 12
    paddd           m1, m2                    ; a3
    paddd          m17, m0                    ; b3 + (1 << 8)
    mova           m16, [t3+r10*4+416*16]     ; b5 + (1 << 7)
    pmaddwd         m1, m3                    ; a3 * src
    pmaddwd         m0, m3, [t3+r10*4+416*12] ; a5 * src
    vpshldd         m3, m25, 16               ; (dst << 16) + (1 << 15)
    psubd          m17, m1                    ; b3 - a3 * src + (1 << 8)
    psubd          m16, m0                    ; b5 - a5 * src + (1 << 7)
    pslld          m17, 7
    palignr    m17{k2}, m16, m16, 1
    vpdpwssd        m3, m17, m26
    packuswb        m3, m3
    vpermb         m16, m27, m3
    mova    [dstq+r10], xm16
    add            r10, 16
    jl .n1_loop
    add           dstq, strideq
    ret

%endif ; ARCH_X86_64