path: root/third_party/dav1d/src/x86/looprestoration_sse.asm

; Copyright © 2018, VideoLAN and dav1d authors
; Copyright © 2018, Two Orioles, LLC
; Copyright © 2018, VideoLabs
; All rights reserved.
;
; Redistribution and use in source and binary forms, with or without
; modification, are permitted provided that the following conditions are met:
;
; 1. Redistributions of source code must retain the above copyright notice, this
;    list of conditions and the following disclaimer.
;
; 2. Redistributions in binary form must reproduce the above copyright notice,
;    this list of conditions and the following disclaimer in the documentation
;    and/or other materials provided with the distribution.
;
; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
; ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
; WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
; ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
; (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
; ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
; SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

%include "config.asm"
%include "ext/x86/x86inc.asm"

SECTION_RODATA 16

wiener_init:   db  6,  7,  6,  7,  6,  7,  6,  7,  0,  0,  0,  0,  2,  4,  2,  4
wiener_shufA:  db  1,  7,  2,  8,  3,  9,  4, 10,  5, 11,  6, 12,  7, 13,  8, 14
wiener_shufB:  db  2,  3,  3,  4,  4,  5,  5,  6,  6,  7,  7,  8,  8,  9,  9, 10
wiener_shufC:  db  6,  5,  7,  6,  8,  7,  9,  8, 10,  9, 11, 10, 12, 11, 13, 12
wiener_shufD:  db  4, -1,  5, -1,  6, -1,  7, -1,  8, -1,  9, -1, 10, -1, 11, -1
wiener_l_shuf: db  0,  0,  0,  0,  0,  1,  2,  3,  4,  5,  6,  7,  8,  9, 10, 11
sgr_lshuf3:    db  0,  0,  0,  1,  2,  3,  4,  5,  6,  7,  8,  9, 10, 11, 12, 13
sgr_lshuf5:    db  0,  0,  0,  0,  1,  2,  3,  4,  5,  6,  7,  8,  9, 10, 11, 12
pb_0to15:      db  0,  1,  2,  3,  4,  5,  6,  7,  8,  9, 10, 11, 12, 13, 14, 15

pb_right_ext_mask: times 24 db 0xff
                   times 8 db 0
pb_1:          times 16 db 1
pb_3:          times 16 db 3
pw_256:        times 8 dw 256
pw_2056:       times 8 dw 2056
pw_m16380:     times 8 dw -16380
pd_4096:       times 4 dd 4096
pd_34816:      times 4 dd 34816
pd_0xffff:     times 4 dd 0xffff
pd_0xf00800a4: times 4 dd 0xf00800a4
pd_0xf00801c7: times 4 dd 0xf00801c7

cextern sgr_x_by_x

SECTION .text

%macro movif64 2 ; dst, src
 %if ARCH_X86_64
    mov             %1, %2
 %endif
%endmacro

%macro movif32 2 ; dst, src
 %if ARCH_X86_32
    mov             %1, %2
 %endif
%endmacro

%if ARCH_X86_32
 %define PIC_base_offset $$

 %macro SETUP_PIC 1-3 1,0 ; PIC_reg, save_PIC_reg, restore_PIC_reg
  %assign pic_reg_stk_off 4
  %xdefine PIC_reg %1
  %if %2 == 1
    mov        [esp], %1
  %endif
    LEA      PIC_reg, PIC_base_offset
  %if %3 == 1
    XCHG_PIC_REG
  %endif
 %endmacro

 %macro XCHG_PIC_REG 0
    mov [esp+pic_reg_stk_off], PIC_reg
    %assign pic_reg_stk_off (pic_reg_stk_off+4) % 8
    mov PIC_reg, [esp+pic_reg_stk_off]
 %endmacro

 %define PIC_sym(sym)   (PIC_reg+(sym)-PIC_base_offset)

%else
 %macro XCHG_PIC_REG 0
 %endmacro

 %define PIC_sym(sym)   (sym)
%endif

%macro WIENER 0
%if ARCH_X86_64
DECLARE_REG_TMP 9, 7, 10, 11, 12, 13, 14 ; ring buffer pointers
cglobal wiener_filter7_8bpc, 4, 15, 16, -384*12-16, dst, stride, left, lpf, \
                                                    w, h, edge, flt, x
    %define tmpstrideq strideq
    %define base 0
    mov           fltq, r6mp
    mov             wd, wm
    movifnidn       hd, hm
    mov          edged, r7m
    movq           m14, [fltq]
    add           lpfq, wq
    movq            m7, [fltq+16]
    add           dstq, wq
    lea             t1, [rsp+wq*2+16]
    mova           m15, [pw_2056]
    neg             wq
%if cpuflag(ssse3)
    pshufb         m14, [wiener_init]
    mova            m8, [wiener_shufA]
    pshufd         m12, m14, q2222  ; x0 x0
    mova            m9, [wiener_shufB]
    pshufd         m13, m14, q3333  ; x1 x2
    mova           m10, [wiener_shufC]
    punpcklqdq     m14, m14         ; x3
    mova           m11, [wiener_shufD]
%else
    mova           m10, [pw_m16380]
    punpcklwd      m14, m14
    pshufd         m11, m14, q0000 ; x0
    pshufd         m12, m14, q1111 ; x1
    pshufd         m13, m14, q2222 ; x2
    pshufd         m14, m14, q3333 ; x3
%endif
%else
DECLARE_REG_TMP 4, 0, _, 5
%if cpuflag(ssse3)
    %define m10         [base+wiener_shufC]
    %define m11         [base+wiener_shufD]
    %define stk_off     96
%else
    %define m10         [base+pw_m16380]
    %define m11         [stk+96]
    %define stk_off     112
%endif
cglobal wiener_filter7_8bpc, 0, 7, 8, -384*12-stk_off, _, x, left, lpf, tmpstride
    %define base        r6-pb_right_ext_mask-21
    %define stk         esp
    %define dstq        leftq
    %define edgeb       byte edged
    %define edged       [stk+ 8]
    %define dstmp       [stk+12]
    %define hd    dword [stk+16]
    %define wq          [stk+20]
    %define strideq     [stk+24]
    %define leftmp      [stk+28]
    %define t2          [stk+32]
    %define t4          [stk+36]
    %define t5          [stk+40]
    %define t6          [stk+44]
    %define m8          [base+wiener_shufA]
    %define m9          [base+wiener_shufB]
    %define m12         [stk+48]
    %define m13         [stk+64]
    %define m14         [stk+80]
    %define m15         [base+pw_2056]
    mov             r1, r6m ; flt
    mov             r0, r0m ; dst
    mov             r4, r4m ; w
    mov           lpfq, lpfm
    mov             r2, r7m ; edge
    mov             r5, r5m ; h
    movq            m3, [r1+ 0]
    movq            m7, [r1+16]
    add             r0, r4
    mov             r1, r1m ; stride
    add           lpfq, r4
    mov          edged, r2
    mov             r2, r2m ; left
    mov          dstmp, r0
    lea             t1, [rsp+r4*2+stk_off]
    mov             hd, r5
    neg             r4
    LEA             r6, pb_right_ext_mask+21
    mov             wq, r4
    mov        strideq, r1
    mov         leftmp, r2
    mov             r4, r1
%if cpuflag(ssse3)
    pshufb          m3, [base+wiener_init]
    pshufd          m1, m3, q2222
    pshufd          m2, m3, q3333
    punpcklqdq      m3, m3
%else
    punpcklwd       m3, m3
    pshufd          m0, m3, q0000
    pshufd          m1, m3, q1111
    pshufd          m2, m3, q2222
    pshufd          m3, m3, q3333
    mova           m11, m0
%endif
    mova           m12, m1
    mova           m13, m2
    mova           m14, m3
%endif
    psllw           m7, 5
    pshufd          m6, m7, q0000 ; y0 y1
    pshufd          m7, m7, q1111 ; y2 y3
    test         edgeb, 4 ; LR_HAVE_TOP
    jz .no_top
    call .h_top
    add           lpfq, strideq
    mov             t6, t1
    mov             t5, t1
    add             t1, 384*2
    call .h_top
    lea             t3, [lpfq+tmpstrideq*4]
    mov           lpfq, dstmp
    add             t3, tmpstrideq
    mov          [rsp], t3 ; below
    mov             t4, t1
    add             t1, 384*2
    call .h
    mov             t3, t1
    mov             t2, t1
    dec             hd
    jz .v1
    add           lpfq, strideq
    add             t1, 384*2
    call .h
    mov             t2, t1
    dec             hd
    jz .v2
    add           lpfq, strideq
    add             t1, 384*2
    call .h
    dec             hd
    jz .v3
.main:
    lea             t0, [t1+384*2]
.main_loop:
    call .hv
    dec             hd
    jnz .main_loop
    test         edgeb, 8 ; LR_HAVE_BOTTOM
    jz .v3
    mov           lpfq, [rsp]
    call .hv_bottom
    add           lpfq, strideq
    call .hv_bottom
.v1:
    call mangle(private_prefix %+ _wiener_filter7_8bpc_ssse3).v
    RET
.no_top:
    lea             t3, [lpfq+tmpstrideq*4]
    mov           lpfq, dstmp
    lea             t3, [t3+tmpstrideq*2]
    mov          [rsp], t3
    call .h
    mov             t6, t1
    mov             t5, t1
    mov             t4, t1
    mov             t3, t1
    mov             t2, t1
    dec             hd
    jz .v1
    add           lpfq, strideq
    add             t1, 384*2
    call .h
    mov             t2, t1
    dec             hd
    jz .v2
    add           lpfq, strideq
    add             t1, 384*2
    call .h
    dec             hd
    jz .v3
    lea             t0, [t1+384*2]
    call .hv
    dec             hd
    jz .v3
    add             t0, 384*8
    call .hv
    dec             hd
    jnz .main
.v3:
    call mangle(private_prefix %+ _wiener_filter7_8bpc_ssse3).v
.v2:
    call mangle(private_prefix %+ _wiener_filter7_8bpc_ssse3).v
    jmp .v1
.extend_right:
    movd            m2, [lpfq-4]
%if ARCH_X86_64
    push            r0
    lea             r0, [pb_right_ext_mask+21]
    movu            m0, [r0+xq+0]
    movu            m1, [r0+xq+8]
    pop             r0
%else
    movu            m0, [r6+xq+0]
    movu            m1, [r6+xq+8]
%endif
%if cpuflag(ssse3)
    pshufb          m2, [base+pb_3]
%else
    punpcklbw       m2, m2
    pshuflw         m2, m2, q3333
    punpcklqdq      m2, m2
%endif
    pand            m4, m0
    pand            m5, m1
    pandn           m0, m2
    pandn           m1, m2
    por             m4, m0
    por             m5, m1
    ret
.h:
    %define stk esp+4 ; offset due to call
    mov             xq, wq
    test         edgeb, 1 ; LR_HAVE_LEFT
    jz .h_extend_left
    movifnidn    leftq, leftmp
    mova            m4, [lpfq+xq]
    movd            m5, [leftq]
    add          leftq, 4
    pslldq          m4, 4
    por             m4, m5
    movifnidn   leftmp, leftq
    jmp .h_main
.h_extend_left:
%if cpuflag(ssse3)
    mova            m4, [lpfq+xq]
    pshufb          m4, [base+wiener_l_shuf]
%else
    mova            m5, [lpfq+xq]
    pshufd          m4, m5, q2103
    punpcklbw       m5, m5
    punpcklwd       m5, m5
    movss           m4, m5
%endif
    jmp .h_main
.h_top:
    mov             xq, wq
    test         edgeb, 1 ; LR_HAVE_LEFT
    jz .h_extend_left
.h_loop:
    movu            m4, [lpfq+xq-4]
.h_main:
    movu            m5, [lpfq+xq+4]
    test         edgeb, 2 ; LR_HAVE_RIGHT
    jnz .h_have_right
    cmp             xd, -18
    jl .h_have_right
    call .extend_right
.h_have_right:
%macro %%h7 0
%if cpuflag(ssse3)
    pshufb          m0, m4, m8
    pmaddubsw       m0, m12
    pshufb          m1, m5, m8
    pmaddubsw       m1, m12
    pshufb          m2, m4, m9
    pmaddubsw       m2, m13
    pshufb          m3, m5, m9
    pmaddubsw       m3, m13
    paddw           m0, m2
    pshufb          m2, m4, m10
    pmaddubsw       m2, m13
    paddw           m1, m3
    pshufb          m3, m5, m10
    pmaddubsw       m3, m13
    pshufb          m4, m11
    paddw           m0, m2
    pmullw          m2, m14, m4
    pshufb          m5, m11
    paddw           m1, m3
    pmullw          m3, m14, m5
    psllw           m4, 7
    psllw           m5, 7
    paddw           m0, m2
    mova            m2, [base+pw_m16380]
    paddw           m1, m3
    paddw           m4, m2
    paddw           m5, m2
    paddsw          m0, m4
    paddsw          m1, m5
%else
    psrldq          m0, m4, 1
    pslldq          m1, m4, 1
    pxor            m3, m3
    punpcklbw       m0, m3
    punpckhbw       m1, m3
    paddw           m0, m1
    pmullw          m0, m11
    psrldq          m1, m4, 2
    pslldq          m2, m4, 2
    punpcklbw       m1, m3
    punpckhbw       m2, m3
    paddw           m1, m2
    pmullw          m1, m12
    paddw           m0, m1
    pshufd          m2, m4, q0321
    punpcklbw       m2, m3
    pmullw          m1, m14, m2
    paddw           m0, m1
    psrldq          m1, m4, 3
    pslldq          m4, 3
    punpcklbw       m1, m3
    punpckhbw       m4, m3
    paddw           m1, m4
    pmullw          m1, m13
    paddw           m0, m1
    psllw           m2, 7
    paddw           m2, m10
    paddsw          m0, m2
    psrldq          m1, m5, 1
    pslldq          m2, m5, 1
    punpcklbw       m1, m3
    punpckhbw       m2, m3
    paddw           m1, m2
    pmullw          m1, m11
    psrldq          m2, m5, 2
    pslldq          m4, m5, 2
    punpcklbw       m2, m3
    punpckhbw       m4, m3
    paddw           m2, m4
    pmullw          m2, m12
    paddw           m1, m2
    pshufd          m4, m5, q0321
    punpcklbw       m4, m3
    pmullw          m2, m14, m4
    paddw           m1, m2
    psrldq          m2, m5, 3
    pslldq          m5, 3
    punpcklbw       m2, m3
    punpckhbw       m5, m3
    paddw           m2, m5
    pmullw          m2, m13
    paddw           m1, m2
    psllw           m4, 7
    paddw           m4, m10
    paddsw          m1, m4
%endif
%endmacro
    %%h7
    psraw           m0, 3
    psraw           m1, 3
    paddw           m0, m15
    paddw           m1, m15
    mova  [t1+xq*2+ 0], m0
    mova  [t1+xq*2+16], m1
    add             xq, 16
    jl .h_loop
    ret
ALIGN function_align
.hv:
    add           lpfq, strideq
    mov             xq, wq
    test         edgeb, 1 ; LR_HAVE_LEFT
    jz .hv_extend_left
    movifnidn    leftq, leftmp
    mova            m4, [lpfq+xq]
    movd            m5, [leftq]
    add          leftq, 4
    pslldq          m4, 4
    por             m4, m5
    movifnidn   leftmp, leftq
    jmp .hv_main
.hv_extend_left:
%if cpuflag(ssse3)
    mova            m4, [lpfq+xq]
    pshufb          m4, [base+wiener_l_shuf]
%else
    mova            m5, [lpfq+xq]
    pshufd          m4, m5, q2103
    punpcklbw       m5, m5
    punpcklwd       m5, m5
    movss           m4, m5
%endif
    jmp .hv_main
.hv_bottom:
    mov             xq, wq
    test         edgeb, 1 ; LR_HAVE_LEFT
    jz .hv_extend_left
.hv_loop:
    movu            m4, [lpfq+xq-4]
.hv_main:
    movu            m5, [lpfq+xq+4]
    test         edgeb, 2 ; LR_HAVE_RIGHT
    jnz .hv_have_right
    cmp             xd, -18
    jl .hv_have_right
    call .extend_right
.hv_have_right:
    %%h7
%if ARCH_X86_64
    mova            m2, [t4+xq*2]
    paddw           m2, [t2+xq*2]
%else
    mov             r2, t4
    mova            m2, [r2+xq*2]
    mov             r2, t2
    paddw           m2, [r2+xq*2]
    mov             r2, t5
%endif
    mova            m3, [t3+xq*2]
%if ARCH_X86_64
    mova            m5, [t5+xq*2]
%else
    mova            m5, [r2+xq*2]
    mov             r2, t6
%endif
    paddw           m5, [t1+xq*2]
    psraw           m0, 3
    psraw           m1, 3
    paddw           m0, m15
    paddw           m1, m15
%if ARCH_X86_64
    paddw           m4, m0, [t6+xq*2]
%else
    paddw           m4, m0, [r2+xq*2]
    mov             r2, t4
%endif
    mova     [t0+xq*2], m0
    punpcklwd       m0, m2, m3
    pmaddwd         m0, m7
    punpckhwd       m2, m3
    pmaddwd         m2, m7
    punpcklwd       m3, m4, m5
    pmaddwd         m3, m6
    punpckhwd       m4, m5
    pmaddwd         m4, m6
    paddd           m0, m3
    mova            m3, [t3+xq*2+16]
    paddd           m4, m2
%if ARCH_X86_64
    mova            m2, [t4+xq*2+16]
    paddw           m2, [t2+xq*2+16]
    mova            m5, [t5+xq*2+16]
%else
    mova            m2, [r2+xq*2+16]
    mov             r2, t2
    paddw           m2, [r2+xq*2+16]
    mov             r2, t5
    mova            m5, [r2+xq*2+16]
    mov             r2, t6
%endif
    paddw           m5, [t1+xq*2+16]
    packuswb        m0, m4
%if ARCH_X86_64
    paddw           m4, m1, [t6+xq*2+16]
%else
    paddw           m4, m1, [r2+xq*2+16]
    mov           dstq, dstmp
%endif
    mova  [t0+xq*2+16], m1
    punpcklwd       m1, m2, m3
    pmaddwd         m1, m7
    punpckhwd       m2, m3
    pmaddwd         m2, m7
    punpcklwd       m3, m4, m5
    pmaddwd         m3, m6
    punpckhwd       m4, m5
    pmaddwd         m4, m6
    paddd           m1, m3
    paddd           m2, m4
    packuswb        m1, m2
    psrlw           m0, 8
    psrlw           m1, 8
    packuswb        m0, m1
    mova     [dstq+xq], m0
    add             xq, 16
    jl .hv_loop
    add           dstq, strideq
%if ARCH_X86_64
    mov             t6, t5
    mov             t5, t4
    mov             t4, t3
    mov             t3, t2
    mov             t2, t1
    mov             t1, t0
    mov             t0, t6
%else
    mov          dstmp, dstq
    mov             r1, t5
    mov             r2, t4
    mov             t6, r1
    mov             t5, r2
    mov             t4, t3
    mov             t3, t2
    mov             t2, t1
    mov             t1, t0
    mov             t0, r1
%endif
    ret
%if cpuflag(ssse3) ; identical in sse2 and ssse3, so share code
.v:
    mov             xq, wq
.v_loop:
%if ARCH_X86_64
    mova            m1, [t4+xq*2]
    paddw           m1, [t2+xq*2]
%else
    mov             r2, t4
    mova            m1, [r2+xq*2]
    mov             r2, t2
    paddw           m1, [r2+xq*2]
    mov             r2, t6
%endif
    mova            m2, [t3+xq*2]
    mova            m4, [t1+xq*2]
%if ARCH_X86_64
    paddw           m3, m4, [t6+xq*2]
    paddw           m4, [t5+xq*2]
%else
    paddw           m3, m4, [r2+xq*2]
    mov             r2, t5
    paddw           m4, [r2+xq*2]
    mov             r2, t4
%endif
    punpcklwd       m0, m1, m2
    pmaddwd         m0, m7
    punpckhwd       m1, m2
    pmaddwd         m1, m7
    punpcklwd       m2, m3, m4
    pmaddwd         m2, m6
    punpckhwd       m3, m4
    pmaddwd         m3, m6
    paddd           m0, m2
    paddd           m1, m3
%if ARCH_X86_64
    mova            m2, [t4+xq*2+16]
    paddw           m2, [t2+xq*2+16]
%else
    mova            m2, [r2+xq*2+16]
    mov             r2, t2
    paddw           m2, [r2+xq*2+16]
    mov             r2, t6
%endif
    mova            m3, [t3+xq*2+16]
    mova            m5, [t1+xq*2+16]
%if ARCH_X86_64
    paddw           m4, m5, [t6+xq*2+16]
    paddw           m5, [t5+xq*2+16]
%else
    paddw           m4, m5, [r2+xq*2+16]
    mov             r2, t5
    paddw           m5, [r2+xq*2+16]
    movifnidn     dstq, dstmp
%endif
    packuswb        m0, m1
    punpcklwd       m1, m2, m3
    pmaddwd         m1, m7
    punpckhwd       m2, m3
    pmaddwd         m2, m7
    punpcklwd       m3, m4, m5
    pmaddwd         m3, m6
    punpckhwd       m4, m5
    pmaddwd         m4, m6
    paddd           m1, m3
    paddd           m2, m4
    packuswb        m1, m2
    psrlw           m0, 8
    psrlw           m1, 8
    packuswb        m0, m1
    mova     [dstq+xq], m0
    add             xq, 16
    jl .v_loop
    add           dstq, strideq
%if ARCH_X86_64
    mov             t6, t5
    mov             t5, t4
%else
    mov          dstmp, dstq
    mov             r1, t5
    mov             r2, t4
    mov             t6, r1
    mov             t5, r2
%endif
    mov             t4, t3
    mov             t3, t2
    mov             t2, t1
    ret
%endif

%if ARCH_X86_64
cglobal wiener_filter5_8bpc, 4, 13, 16, 384*8+16, dst, stride, left, lpf, \
                                                  w, h, edge, flt, x
    mov           fltq, r6mp
    mov             wd, wm
    movifnidn       hd, hm
    mov          edged, r7m
    movq           m14, [fltq]
    add           lpfq, wq
    movq            m7, [fltq+16]
    add           dstq, wq
    mova            m8, [pw_m16380]
    lea             t1, [rsp+wq*2+16]
    mova           m15, [pw_2056]
    neg             wq
%if cpuflag(ssse3)
    pshufb         m14, [wiener_init]
    mova            m9, [wiener_shufB]
    pshufd         m13, m14, q3333  ; x1 x2
    mova           m10, [wiener_shufC]
    punpcklqdq     m14, m14         ; x3
    mova           m11, [wiener_shufD]
    mova           m12, [wiener_l_shuf]
%else
    punpcklwd      m14, m14
    pshufd         m11, m14, q1111 ; x1
    pshufd         m13, m14, q2222 ; x2
    pshufd         m14, m14, q3333 ; x3
%endif
%else
%if cpuflag(ssse3)
    %define stk_off     80
%else
    %define m11         [stk+80]
    %define stk_off     96
%endif
cglobal wiener_filter5_8bpc, 0, 7, 8, -384*8-stk_off, _, x, left, lpf, tmpstride
    %define stk         esp
    %define leftmp      [stk+28]
    %define m8          [base+pw_m16380]
    %define m12         [base+wiener_l_shuf]
    %define m14         [stk+48]
    mov             r1, r6m ; flt
    mov             r0, r0m ; dst
    mov             r4, r4m ; w
    mov           lpfq, lpfm
    mov             r2, r7m ; edge
    mov             r5, r5m ; h
    movq            m2, [r1+ 0]
    movq            m7, [r1+16]
    add             r0, r4
    mov             r1, r1m ; stride
    add           lpfq, r4
    mov          edged, r2
    mov             r2, r2m ; left
    mov          dstmp, r0
    lea             t1, [rsp+r4*2+stk_off]
    mov             hd, r5
    neg             r4
    LEA             r6, pb_right_ext_mask+21
    mov             wq, r4
    mov        strideq, r1
    mov         leftmp, r2
    mov             r4, r1
%if cpuflag(ssse3)
    pshufb          m2, [base+wiener_init]
    pshufd          m1, m2, q3333
    punpcklqdq      m2, m2
%else
    punpcklwd       m2, m2
    pshufd          m0, m2, q1111
    pshufd          m1, m2, q2222
    pshufd          m2, m2, q3333
    mova           m11, m0
%endif
    mova           m13, m1
    mova           m14, m2
%endif
    psllw           m7, 5
    pshufd          m6, m7, q0000 ; __ y1
    pshufd          m7, m7, q1111 ; y2 y3
    test         edgeb, 4 ; LR_HAVE_TOP
    jz .no_top
    call .h_top
    add           lpfq, strideq
    mov             t4, t1
    add             t1, 384*2
    call .h_top
    lea             xq, [lpfq+tmpstrideq*4]
    mov           lpfq, dstmp
    mov             t3, t1
    add             t1, 384*2
    add             xq, tmpstrideq
    mov          [rsp], xq ; below
    call .h
    mov             t2, t1
    dec             hd
    jz .v1
    add           lpfq, strideq
    add             t1, 384*2
    call .h
    dec             hd
    jz .v2
.main:
    mov             t0, t4
.main_loop:
    call .hv
    dec             hd
    jnz .main_loop
    test         edgeb, 8 ; LR_HAVE_BOTTOM
    jz .v2
    mov           lpfq, [rsp]
    call .hv_bottom
    add           lpfq, strideq
    call .hv_bottom
.end:
    RET
.no_top:
    lea             t3, [lpfq+tmpstrideq*4]
    mov           lpfq, dstmp
    lea             t3, [t3+tmpstrideq*2]
    mov          [rsp], t3
    call .h
    mov             t4, t1
    mov             t3, t1
    mov             t2, t1
    dec             hd
    jz .v1
    add           lpfq, strideq
    add             t1, 384*2
    call .h
    dec             hd
    jz .v2
    lea             t0, [t1+384*2]
    call .hv
    dec             hd
    jz .v2
    add             t0, 384*6
    call .hv
    dec             hd
    jnz .main
.v2:
    call mangle(private_prefix %+ _wiener_filter5_8bpc_ssse3).v
    add           dstq, strideq
    mov             t4, t3
    mov             t3, t2
    mov             t2, t1
    movifnidn    dstmp, dstq
.v1:
    call mangle(private_prefix %+ _wiener_filter5_8bpc_ssse3).v
    jmp .end
.h:
    %define stk esp+4
    mov             xq, wq
    test         edgeb, 1 ; LR_HAVE_LEFT
    jz .h_extend_left
    movifnidn    leftq, leftmp
    mova            m4, [lpfq+xq]
    movd            m5, [leftq]
    add          leftq, 4
    pslldq          m4, 4
    por             m4, m5
    movifnidn   leftmp, leftq
    jmp .h_main
.h_extend_left:
%if cpuflag(ssse3)
    mova            m4, [lpfq+xq]
    pshufb          m4, m12
%else
    mova            m5, [lpfq+xq]
    pshufd          m4, m5, q2103
    punpcklbw       m5, m5
    punpcklwd       m5, m5
    movss           m4, m5
%endif
    jmp .h_main
.h_top:
    mov             xq, wq
    test         edgeb, 1 ; LR_HAVE_LEFT
    jz .h_extend_left
.h_loop:
    movu            m4, [lpfq+xq-4]
.h_main:
    movu            m5, [lpfq+xq+4]
    test         edgeb, 2 ; LR_HAVE_RIGHT
    jnz .h_have_right
    cmp             xd, -17
    jl .h_have_right
    call mangle(private_prefix %+ _wiener_filter7_8bpc %+ SUFFIX).extend_right
.h_have_right:
%macro %%h5 0
%if cpuflag(ssse3)
    pshufb          m0, m4, m9
    pmaddubsw       m0, m13
    pshufb          m1, m5, m9
    pmaddubsw       m1, m13
    pshufb          m2, m4, m10
    pmaddubsw       m2, m13
    pshufb          m3, m5, m10
    pmaddubsw       m3, m13
    pshufb          m4, m11
    paddw           m0, m2
    pmullw          m2, m14, m4
    pshufb          m5, m11
    paddw           m1, m3
    pmullw          m3, m14, m5
    psllw           m4, 7
    psllw           m5, 7
    paddw           m4, m8
    paddw           m5, m8
    paddw           m0, m2
    paddw           m1, m3
    paddsw          m0, m4
    paddsw          m1, m5
%else
    psrldq          m0, m4, 2
    pslldq          m1, m4, 2
    pxor            m3, m3
    punpcklbw       m0, m3
    punpckhbw       m1, m3
    paddw           m0, m1
    pmullw          m0, m11
    pshufd          m2, m4, q0321
    punpcklbw       m2, m3
    pmullw          m1, m14, m2
    paddw           m0, m1
    psrldq          m1, m4, 3
    pslldq          m4, 3
    punpcklbw       m1, m3
    punpckhbw       m4, m3
    paddw           m1, m4
    pmullw          m1, m13
    paddw           m0, m1
    psllw           m2, 7
    paddw           m2, m8
    paddsw          m0, m2
    psrldq          m1, m5, 2
    pslldq          m4, m5, 2
    punpcklbw       m1, m3
    punpckhbw       m4, m3
    paddw           m1, m4
    pmullw          m1, m11
    pshufd          m4, m5, q0321
    punpcklbw       m4, m3
    pmullw          m2, m14, m4
    paddw           m1, m2
    psrldq          m2, m5, 3
    pslldq          m5, 3
    punpcklbw       m2, m3
    punpckhbw       m5, m3
    paddw           m2, m5
    pmullw          m2, m13
    paddw           m1, m2
    psllw           m4, 7
    paddw           m4, m8
    paddsw          m1, m4
%endif
%endmacro
    %%h5
    psraw           m0, 3
    psraw           m1, 3
    paddw           m0, m15
    paddw           m1, m15
    mova  [t1+xq*2+ 0], m0
    mova  [t1+xq*2+16], m1
    add             xq, 16
    jl .h_loop
    ret
ALIGN function_align
.hv:
    add           lpfq, strideq
    mov             xq, wq
    test         edgeb, 1 ; LR_HAVE_LEFT
    jz .hv_extend_left
    movifnidn    leftq, leftmp
    mova            m4, [lpfq+xq]
    movd            m5, [leftq]
    add          leftq, 4
    pslldq          m4, 4
    por             m4, m5
    movifnidn   leftmp, leftq
    jmp .hv_main
.hv_extend_left:
%if cpuflag(ssse3)
    mova            m4, [lpfq+xq]
    pshufb          m4, m12
%else
    mova            m5, [lpfq+xq]
    pshufd          m4, m5, q2103
    punpcklbw       m5, m5
    punpcklwd       m5, m5
    movss           m4, m5
%endif
    jmp .hv_main
.hv_bottom:
    mov             xq, wq
    test         edgeb, 1 ; LR_HAVE_LEFT
    jz .hv_extend_left
.hv_loop:
    movu            m4, [lpfq+xq-4]
.hv_main:
    movu            m5, [lpfq+xq+4]
    test         edgeb, 2 ; LR_HAVE_RIGHT
    jnz .hv_have_right
    cmp             xd, -17
    jl .hv_have_right
    call mangle(private_prefix %+ _wiener_filter7_8bpc %+ SUFFIX).extend_right
.hv_have_right:
    %%h5
    mova            m2, [t3+xq*2]
    paddw           m2, [t1+xq*2]
    psraw           m0, 3
    psraw           m1, 3
    paddw           m0, m15
    paddw           m1, m15
%if ARCH_X86_64
    mova            m3, [t2+xq*2]
    paddw           m4, m0, [t4+xq*2]
%else
    mov             r2, t2
    mova            m3, [r2+xq*2]
    mov             r2, t4
    paddw           m4, m0, [r2+xq*2]
%endif
    mova     [t0+xq*2], m0
    punpcklwd       m0, m2, m3
    pmaddwd         m0, m7
    punpckhwd       m2, m3
    pmaddwd         m2, m7
    punpcklwd       m3, m4, m4
    pmaddwd         m3, m6
    punpckhwd       m4, m4
    pmaddwd         m4, m6
    paddd           m0, m3
    paddd           m4, m2
    mova            m2, [t3+xq*2+16]
    paddw           m2, [t1+xq*2+16]
    packuswb        m0, m4
%if ARCH_X86_64
    mova            m3, [t2+xq*2+16]
    paddw           m4, m1, [t4+xq*2+16]
%else
    paddw           m4, m1, [r2+xq*2+16]
    mov             r2, t2
    mova            m3, [r2+xq*2+16]
    mov           dstq, dstmp
%endif
    mova  [t0+xq*2+16], m1
    punpcklwd       m1, m2, m3
    pmaddwd         m1, m7
    punpckhwd       m2, m3
    pmaddwd         m2, m7
    punpcklwd       m3, m4, m4
    pmaddwd         m3, m6
    punpckhwd       m4, m4
    pmaddwd         m4, m6
    paddd           m1, m3
    paddd           m2, m4
    packuswb        m1, m2
    psrlw           m0, 8
    psrlw           m1, 8
    packuswb        m0, m1
    mova     [dstq+xq], m0
    add             xq, 16
    jl .hv_loop
    add           dstq, strideq
    mov             t4, t3
    mov             t3, t2
    mov             t2, t1
    mov             t1, t0
    mov             t0, t4
    movifnidn    dstmp, dstq
    ret
%if cpuflag(ssse3)
.v:
    mov             xq, wq
.v_loop:
    mova            m3, [t1+xq*2]
    paddw           m1, m3, [t3+xq*2]
%if ARCH_X86_64
    mova            m2, [t2+xq*2]
    paddw           m3, [t4+xq*2]
%else
    mov             r2, t2
    mova            m2, [r2+xq*2]
    mov             r2, t4
    paddw           m3, [r2+xq*2]
%endif
    punpcklwd       m0, m1, m2
    pmaddwd         m0, m7
    punpckhwd       m1, m2
    pmaddwd         m1, m7
    punpcklwd       m2, m3
    pmaddwd         m2, m6
    punpckhwd       m3, m3
    pmaddwd         m3, m6
    paddd           m0, m2
    paddd           m1, m3
    mova            m4, [t1+xq*2+16]
    paddw           m2, m4, [t3+xq*2+16]
%if ARCH_X86_64
    mova            m3, [t2+xq*2+16]
    paddw           m4, [t4+xq*2+16]
%else
    paddw           m4, [r2+xq*2+16]
    mov             r2, t2
    mova            m3, [r2+xq*2+16]
    mov           dstq, dstmp
%endif
    packuswb        m0, m1
    punpcklwd       m1, m2, m3
    pmaddwd         m1, m7
    punpckhwd       m2, m3
    pmaddwd         m2, m7
    punpcklwd       m3, m4
    pmaddwd         m3, m6
    punpckhwd       m4, m4
    pmaddwd         m4, m6
    paddd           m1, m3
    paddd           m2, m4
    packuswb        m1, m2
    psrlw           m0, 8
    psrlw           m1, 8
    packuswb        m0, m1
    mova     [dstq+xq], m0
    add             xq, 16
    jl .v_loop
    ret
%endif
%endmacro

INIT_XMM sse2
WIENER

INIT_XMM ssse3
WIENER

;;;;;;;;;;;;;;;;;;;;;;;;;;
;;      self-guided     ;;
;;;;;;;;;;;;;;;;;;;;;;;;;;

%macro GATHERDD 3 ; dst, src, tmp
    movd           %3d, %2
 %if ARCH_X86_64
    movd            %1, [r13+%3]
    pextrw         %3d, %2, 2
    pinsrw          %1, [r13+%3+2], 3
    pextrw         %3d, %2, 4
    pinsrw          %1, [r13+%3+2], 5
    pextrw         %3d, %2, 6
    pinsrw          %1, [r13+%3+2], 7
 %else
    movd            %1, [base+sgr_x_by_x-0xf03+%3]
    pextrw          %3, %2, 2
    pinsrw          %1, [base+sgr_x_by_x-0xf03+%3+2], 3
    pextrw          %3, %2, 4
    pinsrw          %1, [base+sgr_x_by_x-0xf03+%3+2], 5
    pextrw          %3, %2, 6
    pinsrw          %1, [base+sgr_x_by_x-0xf03+%3+2], 7
 %endif
%endmacro

%macro GATHER_X_BY_X 5 ; dst, src0, src1, tmp32, tmp32_restore
 %if ARCH_X86_64
  %define tmp r14
 %else
  %define tmp %4
 %endif
    GATHERDD        %1, %2, tmp
    GATHERDD        %2, %3, tmp
    movif32         %4, %5
    psrld           %1, 24
    psrld           %2, 24
    packssdw        %1, %2
%endmacro

%macro MULLD 3 ; dst, src, tmp
    pmulhuw         %3, %1, %2
    pmullw          %1, %2
    pslld           %3, 16
    paddd           %1, %3
%endmacro

%if ARCH_X86_32
DECLARE_REG_TMP 0, 1, 2, 3, 5
 %if STACK_ALIGNMENT < 16
  %assign extra_stack 5*16
 %else
  %assign extra_stack 3*16
 %endif
cglobal sgr_filter_5x5_8bpc, 1, 7, 8, -400*24-16-extra_stack, \
                             dst, stride, left, lpf, w
 %if STACK_ALIGNMENT < 16
  %define dstm         dword [esp+calloff+16*0+4*6]
  %define stridemp     dword [esp+calloff+16*0+4*7]
  %define leftm        dword [esp+calloff+16*3+4*0]
  %define lpfm         dword [esp+calloff+16*3+4*1]
  %define w0m          dword [esp+calloff+16*3+4*2]
  %define hd           dword [esp+calloff+16*3+4*3]
  %define edgeb         byte [esp+calloff+16*3+4*4]
  %define edged        dword [esp+calloff+16*3+4*4]
  %define leftmp leftm
 %else
  %define w0m wm
  %define hd dword r5m
  %define edgeb  byte r7m
  %define edged dword r7m
 %endif
 %define hvsrcm dword [esp+calloff+4*0]
 %define w1m    dword [esp+calloff+4*1]
 %define t0m    dword [esp+calloff+4*2]
 %define t2m    dword [esp+calloff+4*3]
 %define t3m    dword [esp+calloff+4*4]
 %define t4m    dword [esp+calloff+4*5]
 %define  m8 [base+pb_1]
 %define  m9 [esp+calloff+16*2]
 %define m10 [base+pd_0xf00800a4]
 %define m11 [base+sgr_lshuf5]
 %define m12 [base+pd_34816]
 %define m13 [base+pb_0to15]
 %define r10 r4
 %define base r6-$$
 %assign calloff 0
 %if STACK_ALIGNMENT < 16
    mov        strideq, [rstk+stack_offset+ 8]
    mov          leftq, [rstk+stack_offset+12]
    mov           lpfq, [rstk+stack_offset+16]
    mov             wd, [rstk+stack_offset+20]
    mov           dstm, dstq
    mov       stridemp, strideq
    mov          leftm, leftq
    mov             r1, [rstk+stack_offset+24]
    mov             r2, [rstk+stack_offset+32]
    mov           lpfm, lpfq
    mov             hd, r1
    mov          edged, r2
 %endif
%else
DECLARE_REG_TMP 8, 7, 9, 11, 12
cglobal sgr_filter_5x5_8bpc, 4, 15, 14, -400*24-16, dst, stride, left, lpf, \
                                                    w, h, edge, params
%endif
%if ARCH_X86_64 || STACK_ALIGNMENT >= 16
    mov             wd, wm
%endif
%if ARCH_X86_64
    mov        paramsq, r6mp
    lea            r13, [sgr_x_by_x-0xf03]
    movifnidn       hd, hm
    mov          edged, r7m
    movu            m9, [paramsq]
    add           lpfq, wq
    mova            m8, [pb_1]
    lea             t1, [rsp+wq*2+20]
    mova           m10, [pd_0xf00800a4]
    add           dstq, wq
    lea             t3, [rsp+wq*4+400*12+16]
    mova           m12, [pd_34816]  ; (1 << 11) + (1 << 15)
    lea             t4, [rsp+wq*2+400*20+16]
    pshufhw         m7, m9, q0000
    pshufb          m9, [pw_256]  ; s0
    punpckhqdq      m7, m7        ; w0
    neg             wq
    mova           m13, [pb_0to15]
    pxor            m6, m6
    mova           m11, [sgr_lshuf5]
    psllw           m7, 4
 DEFINE_ARGS dst, stride, left, lpf, _, h, edge, _, _, _, w
 %define lpfm [rsp]
%else
    mov             r1, [rstk+stack_offset+28] ; params
    LEA             r6, $$
    movu            m1, [r1]
    add           lpfm, wq
    lea             t1, [rsp+extra_stack+wq*2+20]
    add           dstq, wq
    lea             t3, [rsp+extra_stack+wq*4+400*12+16]
    mov           dstm, dstq
    lea             t4, [rsp+extra_stack+wq*2+400*20+16]
    mov            t3m, t3
    pshufhw         m7, m1, q0000
    mov            t4m, t4
    pshufb          m1, [base+pw_256] ; s0
    punpckhqdq      m7, m7            ; w0
    psllw           m7, 4
    neg             wq
    mova            m9, m1
    pxor            m6, m6
    mov            w1m, wd
    sub             wd, 2
    mov           lpfq, lpfm
    mov            w0m, wd
 %define strideq r5
%endif
    test         edgeb, 4 ; LR_HAVE_TOP
    jz .no_top
    call .h_top
    add           lpfq, stridemp
    movif32        t2m, t1
    mov             t2, t1
    call .top_fixup
    add             t1, 400*6
    call .h_top
    movif32    strideq, stridemp
    lea            r10, [lpfq+strideq*4]
    mov           lpfq, dstq
    add            r10, strideq
    mov           lpfm, r10 ; below
    movif32        t0m, t2
    mov             t0, t2
    dec             hd
    jz .height1
    or           edged, 16
    call .h
.main:
    add           lpfq, stridemp
    movif32         t4, t4m
    call .hv
    call .prep_n
    sub             hd, 2
    jl .extend_bottom
.main_loop:
    movif32       lpfq, hvsrcm
    add           lpfq, stridemp
%if ARCH_X86_64
    test            hb, hb
%else
    mov             r4, hd
    test            r4, r4
%endif
    jz .odd_height
    call .h
    add           lpfq, stridemp
    call .hv
    movif32       dstq, dstm
    call .n0
    call .n1
    sub             hd, 2
    movif32         t0, t0m
    jge .main_loop
    test         edgeb, 8 ; LR_HAVE_BOTTOM
    jz .extend_bottom
    mov           lpfq, lpfm
    call .h_top
    add           lpfq, stridemp
    call .hv_bottom
.end:
    movif32       dstq, dstm
    call .n0
    call .n1
.end2:
    RET
.height1:
    movif32         t4, t4m
    call .hv
    call .prep_n
    jmp .odd_height_end
.odd_height:
    call .hv
    movif32       dstq, dstm
    call .n0
    call .n1
.odd_height_end:
    call .v
    movif32       dstq, dstm
    call .n0
    jmp .end2
.extend_bottom:
    call .v
    jmp .end
.no_top:
    movif32    strideq, stridemp
    lea            r10, [lpfq+strideq*4]
    mov           lpfq, dstq
    lea            r10, [r10+strideq*2]
    mov           lpfm, r10
    call .h
    lea             t2, [t1+400*6]
    movif32        t2m, t2
    call .top_fixup
    dec             hd
    jz .no_top_height1
    or           edged, 16
    mov             t0, t1
    mov             t1, t2
    movif32        t0m, t0
    jmp .main
.no_top_height1:
    movif32         t3, t3m
    movif32         t4, t4m
    call .v
    call .prep_n
    jmp .odd_height_end
.extend_right:
%assign stack_offset stack_offset+8
%assign calloff 8
    movd            m1, wd
    movd            m3, [lpfq-1]
    pshufb          m1, m6
    pshufb          m3, m6
    psubb           m2, m8, m1
    pcmpgtb         m2, m13
    pand            m5, m2
    pandn           m2, m3
    por             m5, m2
    ret
%assign stack_offset stack_offset-4
%assign calloff 4
.h: ; horizontal boxsum
%if ARCH_X86_64
    lea             wq, [r4-2]
%else
 %define leftq r4
%endif
    test         edgeb, 1 ; LR_HAVE_LEFT
    jz .h_extend_left
    movif32      leftq, leftm
    movddup         m4, [leftq-4]
    movif32         wq, w0m
    mova            m5, [lpfq+wq+2]
    add         leftmp, 4
    palignr         m5, m4, 13
    jmp .h_main
.h_extend_left:
    movif32         wq, w0m
    mova            m5, [lpfq+wq+2]
    pshufb          m5, m11
    jmp .h_main
.h_top:
%if ARCH_X86_64
    lea             wq, [r4-2]
%endif
    test         edgeb, 1 ; LR_HAVE_LEFT
    jz .h_extend_left
    movif32         wq, w0m
.h_loop:
    movu            m5, [lpfq+wq-1]
.h_main:
    test         edgeb, 2 ; LR_HAVE_RIGHT
    jnz .h_have_right
    cmp             wd, -10
    jl .h_have_right
    call .extend_right
.h_have_right:
    punpcklbw       m4, m5, m6
    punpckhbw       m5, m6
    palignr         m2, m5, m4, 2
    paddw           m0, m4, m2
    palignr         m3, m5, m4, 6
    paddw           m0, m3
    punpcklwd       m1, m2, m3
    pmaddwd         m1, m1
    punpckhwd       m2, m3
    pmaddwd         m2, m2
    palignr         m5, m4, 8
    paddw           m0, m5
    punpcklwd       m3, m4, m5
    pmaddwd         m3, m3
    paddd           m1, m3
    punpckhwd       m3, m4, m5
    pmaddwd         m3, m3
    shufps          m4, m5, q2121
    paddw           m0, m4             ; sum
    punpcklwd       m5, m4, m6
    pmaddwd         m5, m5
    punpckhwd       m4, m6
    pmaddwd         m4, m4
    paddd           m2, m3
    test         edgeb, 16             ; y > 0
    jz .h_loop_end
    paddw           m0, [t1+wq*2+400*0]
    paddd           m1, [t1+wq*2+400*2]
    paddd           m2, [t1+wq*2+400*4]
.h_loop_end:
    paddd           m1, m5             ; sumsq
    paddd           m2, m4
    mova [t1+wq*2+400*0], m0
    mova [t1+wq*2+400*2], m1
    mova [t1+wq*2+400*4], m2
    add             wq, 8
    jl .h_loop
    ret
.top_fixup:
%if ARCH_X86_64
    lea             wq, [r4-2]
%else
    mov             wd, w0m
%endif
.top_fixup_loop: ; the sums of the first row needs to be doubled
    mova            m0, [t1+wq*2+400*0]
    mova            m1, [t1+wq*2+400*2]
    mova            m2, [t1+wq*2+400*4]
    paddw           m0, m0
    paddd           m1, m1
    paddd           m2, m2
    mova [t2+wq*2+400*0], m0
    mova [t2+wq*2+400*2], m1
    mova [t2+wq*2+400*4], m2
    add             wq, 8
    jl .top_fixup_loop
    ret
ALIGN function_align
.hv: ; horizontal boxsum + vertical boxsum + ab
%if ARCH_X86_64
    lea             wq, [r4-2]
%else
    mov         hvsrcm, lpfq
%endif
    test         edgeb, 1 ; LR_HAVE_LEFT
    jz .hv_extend_left
    movif32      leftq, leftm
    movddup         m4, [leftq-4]
    movif32         wq, w0m
    mova            m5, [lpfq+wq+2]
    add         leftmp, 4
    palignr         m5, m4, 13
    jmp .hv_main
.hv_extend_left:
    movif32         wq, w0m
    mova            m5, [lpfq+wq+2]
    pshufb          m5, m11
    jmp .hv_main
.hv_bottom:
%if ARCH_X86_64
    lea             wq, [r4-2]
%else
    mov         hvsrcm, lpfq
%endif
    test         edgeb, 1 ; LR_HAVE_LEFT
    jz .hv_extend_left
    movif32         wq, w0m
%if ARCH_X86_32
    jmp .hv_loop_start
%endif
.hv_loop:
    movif32       lpfq, hvsrcm
.hv_loop_start:
    movu            m5, [lpfq+wq-1]
.hv_main:
    test         edgeb, 2 ; LR_HAVE_RIGHT
    jnz .hv_have_right
    cmp             wd, -10
    jl .hv_have_right
    call .extend_right
.hv_have_right:
    movif32         t3, hd
    punpcklbw       m4, m5, m6
    punpckhbw       m5, m6
    palignr         m3, m5, m4, 2
    paddw           m0, m4, m3
    palignr         m1, m5, m4, 6
    paddw           m0, m1
    punpcklwd       m2, m3, m1
    pmaddwd         m2, m2
    punpckhwd       m3, m1
    pmaddwd         m3, m3
    palignr         m5, m4, 8
    paddw           m0, m5
    punpcklwd       m1, m4, m5
    pmaddwd         m1, m1
    paddd           m2, m1
    punpckhwd       m1, m4, m5
    pmaddwd         m1, m1
    shufps          m4, m5, q2121
    paddw           m0, m4            ; h sum
    punpcklwd       m5, m4, m6
    pmaddwd         m5, m5
    punpckhwd       m4, m6
    pmaddwd         m4, m4
    paddd           m3, m1
    paddd           m2, m5            ; h sumsq
    paddd           m3, m4
    paddw           m1, m0, [t1+wq*2+400*0]
    paddd           m4, m2, [t1+wq*2+400*2]
    paddd           m5, m3, [t1+wq*2+400*4]
%if ARCH_X86_64
    test            hd, hd
%else
    test            t3, t3
%endif
    jz .hv_last_row
.hv_main2:
    paddw           m1, [t2+wq*2+400*0] ; hv sum
    paddd           m4, [t2+wq*2+400*2] ; hv sumsq
    paddd           m5, [t2+wq*2+400*4]
    mova [t0+wq*2+400*0], m0
    pslld           m0, m4, 4
    mova [t0+wq*2+400*2], m2
    mova [t0+wq*2+400*4], m3
    pslld           m2, m4, 3
    paddd           m4, m0
    pslld           m0, m5, 4
    paddd           m4, m2             ; a * 25
    pslld           m2, m5, 3
    paddd           m5, m0
    paddd           m5, m2
    punpcklwd       m0, m1, m6         ; b
    punpckhwd       m1, m6
    pmaddwd         m2, m0, m0         ; b * b
    pmaddwd         m3, m1, m1
    psubd           m4, m2             ; p
    psubd           m5, m3
    MULLD           m4, m9, m2         ; p * s
    MULLD           m5, m9, m2
    pmaddwd         m0, m10            ; b * 164
    pmaddwd         m1, m10
    paddusw         m4, m10
    paddusw         m5, m10
    psrld           m4, 20             ; min(z, 255)
    movif32         t3, t3m
    psrld           m5, 20
    GATHER_X_BY_X   m3, m4, m5, t2, t2m
    punpcklwd       m4, m3, m3
    punpckhwd       m5, m3, m3
    MULLD           m0, m4, m2
    MULLD           m1, m5, m2
    paddd           m0, m12            ; x * b * 164 + (1 << 11) + (1 << 15)
    paddd           m1, m12
    mova   [t4+wq*2+4], m3
    psrld           m0, 12             ; b
    psrld           m1, 12
    mova  [t3+wq*4+ 8], m0
    mova  [t3+wq*4+24], m1
    add             wq, 8
    jl .hv_loop
    mov             t2, t1
    mov             t1, t0
    mov             t0, t2
    movif32        t2m, t2
    movif32        t0m, t0
    ret
.hv_last_row: ; esoteric edge case for odd heights
    mova [t1+wq*2+400*0], m1
    paddw             m1, m0
    mova [t1+wq*2+400*2], m4
    paddd             m4, m2
    mova [t1+wq*2+400*4], m5
    paddd             m5, m3
    jmp .hv_main2
.v: ; vertical boxsum + ab
%if ARCH_X86_64
    lea             wq, [r4-2]
%else
    mov             wd, w0m
%endif
.v_loop:
    mova            m0, [t1+wq*2+400*0]
    mova            m2, [t1+wq*2+400*2]
    mova            m3, [t1+wq*2+400*4]
    paddw           m1, m0, [t2+wq*2+400*0]
    paddd           m4, m2, [t2+wq*2+400*2]
    paddd           m5, m3, [t2+wq*2+400*4]
    paddw           m0, m0
    paddd           m2, m2
    paddd           m3, m3
    paddw           m1, m0             ; hv sum
    paddd           m4, m2             ; hv sumsq
    pslld           m0, m4, 4
    paddd           m5, m3
    pslld           m2, m4, 3
    paddd           m4, m0
    pslld           m0, m5, 4
    paddd           m4, m2             ; a * 25
    pslld           m2, m5, 3
    paddd           m5, m0
    paddd           m5, m2
    punpcklwd       m0, m1, m6
    punpckhwd       m1, m6
    pmaddwd         m2, m0, m0         ; b * b
    pmaddwd         m3, m1, m1
    psubd           m4, m2             ; p
    psubd           m5, m3
    MULLD           m4, m9, m2         ; p * s
    MULLD           m5, m9, m2
    pmaddwd         m0, m10            ; b * 164
    pmaddwd         m1, m10
    paddusw         m4, m10
    paddusw         m5, m10
    psrld           m4, 20             ; min(z, 255)
    psrld           m5, 20
    GATHER_X_BY_X   m3, m4, m5, t2, t2m
    punpcklwd       m4, m3, m3
    punpckhwd       m5, m3, m3
    MULLD           m0, m4, m2
    MULLD           m1, m5, m2
    paddd           m0, m12            ; x * b * 164 + (1 << 11) + (1 << 15)
    paddd           m1, m12
    mova   [t4+wq*2+4], m3
    psrld           m0, 12             ; b
    psrld           m1, 12
    mova  [t3+wq*4+ 8], m0
    mova  [t3+wq*4+24], m1
    add             wq, 8
    jl .v_loop
    ret
.prep_n: ; initial neighbor setup
    movif64         wq, r4
    movif32         wd, w1m
.prep_n_loop:
    movu            m0, [t4+wq*2+ 2]
    movu            m3, [t4+wq*2+ 4]
    movu            m1, [t3+wq*4+ 4]
    movu            m4, [t3+wq*4+ 8]
    movu            m2, [t3+wq*4+20]
    movu            m5, [t3+wq*4+24]
    paddw           m3, m0
    paddd           m4, m1
    paddd           m5, m2
    paddw           m3, [t4+wq*2+ 0]
    paddd           m4, [t3+wq*4+ 0]
    paddd           m5, [t3+wq*4+16]
    paddw           m0, m3
    psllw           m3, 2
    paddd           m1, m4
    pslld           m4, 2
    paddd           m2, m5
    pslld           m5, 2
    paddw           m0, m3             ; a 565
    paddd           m1, m4             ; b 565
    paddd           m2, m5
    mova [t4+wq*2+400*2+ 0], m0
    mova [t3+wq*4+400*4+ 0], m1
    mova [t3+wq*4+400*4+16], m2
    add             wq, 8
    jl .prep_n_loop
    ret
ALIGN function_align
.n0: ; neighbor + output (even rows)
    movif64         wq, r4
    movif32         wd, w1m
.n0_loop:
    movu            m0, [t4+wq*2+ 2]
    movu            m3, [t4+wq*2+ 4]
    movu            m1, [t3+wq*4+ 4]
    movu            m4, [t3+wq*4+ 8]
    movu            m2, [t3+wq*4+20]
    movu            m5, [t3+wq*4+24]
    paddw           m3, m0
    paddd           m4, m1
    paddd           m5, m2
    paddw           m3, [t4+wq*2+ 0]
    paddd           m4, [t3+wq*4+ 0]
    paddd           m5, [t3+wq*4+16]
    paddw           m0, m3
    psllw           m3, 2
    paddd           m1, m4
    pslld           m4, 2
    paddd           m2, m5
    pslld           m5, 2
    paddw           m0, m3             ; a 565
    paddd           m1, m4             ; b 565
    paddd           m2, m5
    paddw           m3, m0, [t4+wq*2+400*2+ 0]
    paddd           m4, m1, [t3+wq*4+400*4+ 0]
    paddd           m5, m2, [t3+wq*4+400*4+16]
    mova [t4+wq*2+400*2+ 0], m0
    mova [t3+wq*4+400*4+ 0], m1
    mova [t3+wq*4+400*4+16], m2
    movq            m0, [dstq+wq]
    punpcklbw       m0, m6
    punpcklwd       m1, m0, m6          ; src
    punpcklwd       m2, m3, m6          ; a
    pmaddwd         m2, m1              ; a * src
    punpckhwd       m1, m0, m6
    punpckhwd       m3, m6
    pmaddwd         m3, m1
    psubd           m4, m2              ; b - a * src + (1 << 8)
    psubd           m5, m3
    psrad           m4, 9
    psrad           m5, 9
    packssdw        m4, m5
    pmulhrsw        m4, m7
    paddw           m0, m4
    packuswb        m0, m0
    movq     [dstq+wq], m0
    add             wq, 8
    jl .n0_loop
    add           dstq, stridemp
    ret
ALIGN function_align
.n1: ; neighbor + output (odd rows)
    movif64         wq, r4
    movif32         wd, w1m
.n1_loop:
    movq            m0, [dstq+wq]
    mova            m3, [t4+wq*2+400*2+ 0]
    mova            m4, [t3+wq*4+400*4+ 0]
    mova            m5, [t3+wq*4+400*4+16]
    punpcklbw       m0, m6
    punpcklwd       m1, m0, m6          ; src
    punpcklwd       m2, m3, m6          ; a
    pmaddwd         m2, m1              ; a * src
    punpckhwd       m1, m0, m6
    punpckhwd       m3, m6
    pmaddwd         m3, m1
    psubd           m4, m2              ; b - a * src + (1 << 7)
    psubd           m5, m3
    psrad           m4, 8
    psrad           m5, 8
    packssdw        m4, m5
    pmulhrsw        m4, m7
    paddw           m0, m4
    packuswb        m0, m0
    movq     [dstq+wq], m0
    add             wq, 8
    jl .n1_loop
    add           dstq, stridemp
    movif32       dstm, dstq
    ret

%if ARCH_X86_32
 %if STACK_ALIGNMENT < 16
  %assign extra_stack 4*16
 %else
  %assign extra_stack 2*16
 %endif
cglobal sgr_filter_3x3_8bpc, 1, 7, 8, -400*42-16-extra_stack, \
                             dst, stride, left, lpf, w
 %if STACK_ALIGNMENT < 16
  %define dstm         dword [esp+calloff+16*2+4*0]
  %define stridemp     dword [esp+calloff+16*2+4*1]
  %define leftm        dword [esp+calloff+16*2+4*2]
  %define lpfm         dword [esp+calloff+16*2+4*3]
  %define w0m          dword [esp+calloff+16*2+4*4]
  %define hd           dword [esp+calloff+16*2+4*5]
  %define edgeb         byte [esp+calloff+16*2+4*6]
  %define edged        dword [esp+calloff+16*2+4*6]
  %define leftmp leftm
 %else
  %define w0m wm
  %define hd dword r5m
  %define edgeb  byte r7m
  %define edged dword r7m
 %endif
 %define hvsrcm dword [esp+calloff+4*0]
 %define w1m    dword [esp+calloff+4*1]
 %define t3m    dword [esp+calloff+4*2]
 %define t4m    dword [esp+calloff+4*3]
 %define  m8 [base+pb_0to15]
 %define  m9 [esp+calloff+16*1]
 %define m10 [base+pd_0xf00801c7]
 %define m11 [base+pd_34816]
 %define m12 m6
 %define m13 [base+sgr_lshuf3]
 %define base r6-$$
 %assign calloff 0
 %if STACK_ALIGNMENT < 16
    mov        strideq, [rstk+stack_offset+ 8]
    mov          leftq, [rstk+stack_offset+12]
    mov           lpfq, [rstk+stack_offset+16]
    mov             wd, [rstk+stack_offset+20]
    mov           dstm, dstq
    mov       stridemp, strideq
    mov          leftm, leftq
    mov             r1, [rstk+stack_offset+24]
    mov             r2, [rstk+stack_offset+32]
    mov           lpfm, lpfq
    mov             hd, r1
    mov          edged, r2
 %endif
%else
cglobal sgr_filter_3x3_8bpc, 4, 15, 14, -400*42-8, dst, stride, left, lpf, \
                                                   w, h, edge, params
%endif
%if ARCH_X86_64 || STACK_ALIGNMENT >= 16
    mov             wd, wm
%endif
%if ARCH_X86_64
    mov        paramsq, r6mp
    lea            r13, [sgr_x_by_x-0xf03]
    mov             hd, hm
    mov          edged, r7m
    movq            m9, [paramsq+4]
    add           lpfq, wq
    lea             t1, [rsp+wq*2+12]
    mova            m8, [pb_0to15]
    add           dstq, wq
    lea             t3, [rsp+wq*4+400*12+8]
    mova           m10, [pd_0xf00801c7]
    lea             t4, [rsp+wq*2+400*32+8]
    mova           m11, [pd_34816]
    pshuflw         m7, m9, q3333
    pshufb          m9, [pw_256]  ; s1
    punpcklqdq      m7, m7        ; w1
    neg             wq
    pxor            m6, m6
    mova           m13, [sgr_lshuf3]
    psllw           m7, 4
 DEFINE_ARGS dst, stride, left, lpf, _, h, edge, _, _, _, w
 %define lpfm [rsp]
%else
    mov             r1, [rstk+stack_offset+28] ; params
    LEA             r6, $$
    movq            m1, [r1+4]
    add           lpfm, wq
    lea             t1, [rsp+extra_stack+wq*2+20]
    add           dstq, wq
    lea             t3, [rsp+extra_stack+wq*4+400*12+16]
    mov           dstm, dstq
    lea             t4, [rsp+extra_stack+wq*2+400*32+16]
    mov            t3m, t3
    pshuflw         m7, m1, q3333
    mov            t4m, t4
    pshufb          m1, [base+pw_256] ; s1
    punpcklqdq      m7, m7            ; w1
    psllw           m7, 4
    neg             wq
    mova            m9, m1
    pxor            m6, m6
    mov            w1m, wd
    sub             wd, 2
    mov           lpfq, lpfm
    mov            w0m, wd
 %define strideq r5
%endif
    test         edgeb, 4 ; LR_HAVE_TOP
    jz .no_top
    call .h_top
    add           lpfq, stridemp
    mov             t2, t1
    add             t1, 400*6
    call .h_top
    movif32    strideq, stridemp
    lea            r10, [lpfq+strideq*4]
    mov           lpfq, dstq
    add            r10, strideq
    mov           lpfm, r10 ; below
    movif32         t4, t4m
    call .hv0
.main:
    dec             hd
    jz .height1
    movif32       lpfq, hvsrcm
    add           lpfq, stridemp
    call .hv1
    call .prep_n
    sub             hd, 2
    jl .extend_bottom
.main_loop:
    movif32       lpfq, hvsrcm
    add           lpfq, stridemp
    call .hv0
%if ARCH_X86_64
    test            hb, hb
%else
    mov             r4, hd
    test            r4, r4
%endif
    jz .odd_height
    movif32       lpfq, hvsrcm
    add           lpfq, stridemp
    call .hv1
    call .n0
    call .n1
    sub             hd, 2
    jge .main_loop
    test         edgeb, 8 ; LR_HAVE_BOTTOM
    jz .extend_bottom
    mov           lpfq, lpfm
    call .hv0_bottom
    movif32       lpfq, hvsrcm
    add           lpfq, stridemp
    call .hv1_bottom
.end:
    call .n0
    call .n1
.end2:
    RET
.height1:
    call .v1
    call .prep_n
    jmp .odd_height_end
.odd_height:
    call .v1
    call .n0
    call .n1
.odd_height_end:
    call .v0
    call .v1
    call .n0
    jmp .end2
.extend_bottom:
    call .v0
    call .v1
    jmp .end
.no_top:
    movif32    strideq, stridemp
    lea            r10, [lpfq+strideq*4]
    mov           lpfq, dstq
    lea            r10, [r10+strideq*2]
    mov           lpfm, r10
    call .h
%if ARCH_X86_64
    lea             wq, [r4-2]
%else
    mov             wq, w0m
    mov         hvsrcm, lpfq
%endif
    lea             t2, [t1+400*6]
.top_fixup_loop:
    mova            m0, [t1+wq*2+400*0]
    mova            m1, [t1+wq*2+400*2]
    mova            m2, [t1+wq*2+400*4]
    mova [t2+wq*2+400*0], m0
    mova [t2+wq*2+400*2], m1
    mova [t2+wq*2+400*4], m2
    add             wq, 8
    jl .top_fixup_loop
    movif32         t3, t3m
    movif32         t4, t4m
    call .v0
    jmp .main
.extend_right:
%assign stack_offset stack_offset+8
%assign calloff 8
    movd            m0, [lpfq-1]
    movd            m1, wd
    mova            m3, m8
    pshufb          m0, m6
    pshufb          m1, m6
    mova            m2, m6
    psubb           m2, m1
    pcmpgtb         m2, m3
    pand            m5, m2
    pandn           m2, m0
    por             m5, m2
    ret
%assign stack_offset stack_offset-4
%assign calloff 4
.h: ; horizontal boxsum
%if ARCH_X86_64
    lea             wq, [r4-2]
%else
 %define leftq r4
%endif
    test         edgeb, 1 ; LR_HAVE_LEFT
    jz .h_extend_left
    movif32      leftq, leftm
    movddup         m4, [leftq-4]
    movif32         wq, w0m
    mova            m5, [lpfq+wq+2]
    add         leftmp, 4
    palignr         m5, m4, 14
    jmp .h_main
.h_extend_left:
    movif32         wq, w0m
    mova            m5, [lpfq+wq+2]
    pshufb          m5, m13
    jmp .h_main
.h_top:
%if ARCH_X86_64
    lea             wq, [r4-2]
%endif
    test         edgeb, 1 ; LR_HAVE_LEFT
    jz .h_extend_left
    movif32         wq, w0m
.h_loop:
    movu            m5, [lpfq+wq]
.h_main:
    test         edgeb, 2 ; LR_HAVE_RIGHT
    jnz .h_have_right
    cmp             wd, -9
    jl .h_have_right
    call .extend_right
.h_have_right:
    punpcklbw       m4, m5, m6
    punpckhbw       m5, m6
    palignr         m0, m5, m4, 2
    paddw           m1, m4, m0
    punpcklwd       m2, m4, m0
    pmaddwd         m2, m2
    punpckhwd       m3, m4, m0
    pmaddwd         m3, m3
    palignr         m5, m4, 4
    paddw           m1, m5             ; sum
    punpcklwd       m4, m5, m6
    pmaddwd         m4, m4
    punpckhwd       m5, m6
    pmaddwd         m5, m5
    paddd           m2, m4             ; sumsq
    paddd           m3, m5
    mova [t1+wq*2+400*0], m1
    mova [t1+wq*2+400*2], m2
    mova [t1+wq*2+400*4], m3
    add             wq, 8
    jl .h_loop
    ret
ALIGN function_align
.hv0: ; horizontal boxsum + vertical boxsum + ab (even rows)
%if ARCH_X86_64
    lea             wq, [r4-2]
%else
    mov         hvsrcm, lpfq
%endif
    test         edgeb, 1 ; LR_HAVE_LEFT
    jz .hv0_extend_left
    movif32      leftq, leftm
    movddup         m4, [leftq-4]
    movif32         wq, w0m
    mova            m5, [lpfq+wq+2]
    add         leftmp, 4
    palignr         m5, m4, 14
    jmp .hv0_main
.hv0_extend_left:
    movif32         wq, w0m
    mova            m5, [lpfq+wq+2]
    pshufb          m5, m13
    jmp .hv0_main
.hv0_bottom:
%if ARCH_X86_64
    lea             wq, [r4-2]
%else
    mov         hvsrcm, lpfq
%endif
    test         edgeb, 1 ; LR_HAVE_LEFT
    jz .hv0_extend_left
    movif32         wq, w0m
%if ARCH_X86_32
    jmp .hv0_loop_start
%endif
.hv0_loop:
    movif32       lpfq, hvsrcm
.hv0_loop_start:
    movu            m5, [lpfq+wq]
.hv0_main:
    test         edgeb, 2 ; LR_HAVE_RIGHT
    jnz .hv0_have_right
    cmp             wd, -9
    jl .hv0_have_right
    call .extend_right
.hv0_have_right:
    punpcklbw       m4, m5, m6
    punpckhbw       m5, m6
    palignr         m0, m5, m4, 2
    paddw           m1, m4, m0
    punpcklwd       m2, m4, m0
    pmaddwd         m2, m2
    punpckhwd       m3, m4, m0
    pmaddwd         m3, m3
    palignr         m5, m4, 4
    paddw           m1, m5             ; sum
    punpcklwd       m4, m5, m6
    pmaddwd         m4, m4
    punpckhwd       m5, m6
    pmaddwd         m5, m5
    paddd           m2, m4             ; sumsq
    paddd           m3, m5
    paddw           m0, m1, [t1+wq*2+400*0]
    paddd           m4, m2, [t1+wq*2+400*2]
    paddd           m5, m3, [t1+wq*2+400*4]
    mova [t1+wq*2+400*0], m1
    mova [t1+wq*2+400*2], m2
    mova [t1+wq*2+400*4], m3
    paddw           m1, m0, [t2+wq*2+400*0]
    paddd           m2, m4, [t2+wq*2+400*2]
    paddd           m3, m5, [t2+wq*2+400*4]
    mova [t2+wq*2+400*0], m0
    mova [t2+wq*2+400*2], m4
    mova [t2+wq*2+400*4], m5
    pslld           m4, m2, 3
    pslld           m5, m3, 3
    paddd           m4, m2             ; a * 9
    paddd           m5, m3
    punpcklwd       m0, m1, m6         ; b
    pmaddwd         m2, m0, m0         ; b * b
    punpckhwd       m1, m6
    pmaddwd         m3, m1, m1
    psubd           m4, m2             ; p
    psubd           m5, m3
    MULLD           m4, m9, m12        ; p * s
    MULLD           m5, m9, m12
    pmaddwd         m0, m10            ; b * 455
    pmaddwd         m1, m10
    paddusw         m4, m10
    paddusw         m5, m10
    psrld           m4, 20             ; min(z, 255)
    movif32         t3, t3m
    psrld           m5, 20
    GATHER_X_BY_X   m3, m4, m5, r0, dstm
    punpcklwd       m4, m3, m3
    punpckhwd       m5, m3, m3
    MULLD           m0, m4, m12
    MULLD           m1, m5, m12
%if ARCH_X86_32
    pxor            m6, m6
%endif
    paddd           m0, m11            ; x * b * 455 + (1 << 11) + (1 << 15)
    paddd           m1, m11
    mova   [t4+wq*2+4], m3
    psrld           m0, 12
    psrld           m1, 12
    mova  [t3+wq*4+ 8], m0
    mova  [t3+wq*4+24], m1
    add             wq, 8
    jl .hv0_loop
    ret
ALIGN function_align
.hv1: ; horizontal boxsums + vertical boxsums + ab (odd rows)
%if ARCH_X86_64
    lea             wq, [r4-2]
%else
    mov         hvsrcm, lpfq
%endif
    test         edgeb, 1 ; LR_HAVE_LEFT
    jz .hv1_extend_left
    movif32      leftq, leftm
    movddup         m4, [leftq-4]
    movif32         wq, w0m
    mova            m5, [lpfq+wq+2]
    add         leftmp, 4
    palignr         m5, m4, 14
    jmp .hv1_main
.hv1_extend_left:
    movif32         wq, w0m
    mova            m5, [lpfq+wq+2]
    pshufb          m5, m13
    jmp .hv1_main
.hv1_bottom:
%if ARCH_X86_64
    lea             wq, [r4-2]
%else
    mov         hvsrcm, lpfq
%endif
    test         edgeb, 1 ; LR_HAVE_LEFT
    jz .hv1_extend_left
    movif32         wq, w0m
%if ARCH_X86_32
    jmp .hv1_loop_start
%endif
.hv1_loop:
    movif32       lpfq, hvsrcm
.hv1_loop_start:
    movu            m5, [lpfq+wq]
.hv1_main:
    test         edgeb, 2 ; LR_HAVE_RIGHT
    jnz .hv1_have_right
    cmp             wd, -9
    jl .hv1_have_right
    call .extend_right
.hv1_have_right:
    punpcklbw       m4, m5, m6
    punpckhbw       m5, m6
    palignr         m1, m5, m4, 2
    paddw           m0, m4, m1
    punpcklwd       m2, m4, m1
    pmaddwd         m2, m2
    punpckhwd       m3, m4, m1
    pmaddwd         m3, m3
    palignr         m5, m4, 4
    paddw           m0, m5             ; h sum
    punpcklwd       m1, m5, m6
    pmaddwd         m1, m1
    punpckhwd       m5, m6
    pmaddwd         m5, m5
    paddd           m2, m1             ; h sumsq
    paddd           m3, m5
    paddw           m1, m0, [t2+wq*2+400*0]
    paddd           m4, m2, [t2+wq*2+400*2]
    paddd           m5, m3, [t2+wq*2+400*4]
    mova [t2+wq*2+400*0], m0
    mova [t2+wq*2+400*2], m2
    mova [t2+wq*2+400*4], m3
    pslld           m2, m4, 3
    pslld           m3, m5, 3
    paddd           m4, m2             ; a * 9
    paddd           m5, m3
    punpcklwd       m0, m1, m6         ; b
    pmaddwd         m2, m0, m0         ; b * b
    punpckhwd       m1, m6
    pmaddwd         m3, m1, m1
    psubd           m4, m2             ; p
    psubd           m5, m3
    MULLD           m4, m9, m12        ; p * s
    MULLD           m5, m9, m12
    pmaddwd         m0, m10            ; b * 455
    pmaddwd         m1, m10
    paddusw         m4, m10
    paddusw         m5, m10
    psrld           m4, 20             ; min(z, 255)
    movif32         t3, t3m
    psrld           m5, 20
    GATHER_X_BY_X   m3, m4, m5, r0, dstm
    punpcklwd       m4, m3, m3
    punpckhwd       m5, m3, m3
    MULLD           m0, m4, m12
    MULLD           m1, m5, m12
%if ARCH_X86_32
    pxor            m6, m6
%endif
    paddd           m0, m11            ; x * b * 455 + (1 << 11) + (1 << 15)
    paddd           m1, m11
    mova [t4+wq*2+400*2 +4], m3
    psrld           m0, 12
    psrld           m1, 12
    mova [t3+wq*4+400*4+ 8], m0
    mova [t3+wq*4+400*4+24], m1
    add             wq, 8
    jl .hv1_loop
    mov            r10, t2
    mov             t2, t1
    mov             t1, r10
    ret
.v0: ; vertical boxsums + ab (even rows)
%if ARCH_X86_64
    lea             wq, [r4-2]
%else
    mov             wd, w0m
%endif
.v0_loop:
    mova            m0, [t1+wq*2+400*0]
    mova            m4, [t1+wq*2+400*2]
    mova            m5, [t1+wq*2+400*4]
    paddw           m0, m0
    paddd           m4, m4
    paddd           m5, m5
    paddw           m1, m0, [t2+wq*2+400*0]
    paddd           m2, m4, [t2+wq*2+400*2]
    paddd           m3, m5, [t2+wq*2+400*4]
    mova [t2+wq*2+400*0], m0
    mova [t2+wq*2+400*2], m4
    mova [t2+wq*2+400*4], m5
    pslld           m4, m2, 3
    pslld           m5, m3, 3
    paddd           m4, m2             ; a * 9
    paddd           m5, m3
    punpcklwd       m0, m1, m6         ; b
    pmaddwd         m2, m0, m0         ; b * b
    punpckhwd       m1, m6
    pmaddwd         m3, m1, m1
    psubd           m4, m2             ; p
    psubd           m5, m3
    MULLD           m4, m9, m12        ; p * s
    MULLD           m5, m9, m12
    pmaddwd         m0, m10            ; b * 455
    pmaddwd         m1, m10
    paddusw         m4, m10
    paddusw         m5, m10
    psrld           m4, 20             ; min(z, 255)
    psrld           m5, 20
    GATHER_X_BY_X   m3, m4, m5, r0, dstm
    punpcklwd       m4, m3, m3
    punpckhwd       m5, m3, m3
    MULLD           m0, m4, m12
    MULLD           m1, m5, m12
%if ARCH_X86_32
    pxor            m6, m6
%endif
    paddd           m0, m11            ; x * b * 455 + (1 << 11) + (1 << 15)
    paddd           m1, m11
    mova   [t4+wq*2+4], m3
    psrld           m0, 12
    psrld           m1, 12
    mova  [t3+wq*4+ 8], m0
    mova  [t3+wq*4+24], m1
    add             wq, 8
    jl .v0_loop
    ret
.v1: ; vertical boxsums + ab (odd rows)
%if ARCH_X86_64
    lea             wq, [r4-2]
%else
    mov             wd, w0m
%endif
.v1_loop:
    mova            m0, [t1+wq*2+400*0]
    mova            m4, [t1+wq*2+400*2]
    mova            m5, [t1+wq*2+400*4]
    paddw           m1, m0, [t2+wq*2+400*0]
    paddd           m2, m4, [t2+wq*2+400*2]
    paddd           m3, m5, [t2+wq*2+400*4]
    mova [t2+wq*2+400*0], m0
    mova [t2+wq*2+400*2], m4
    mova [t2+wq*2+400*4], m5
    pslld           m4, m2, 3
    pslld           m5, m3, 3
    paddd           m4, m2             ; a * 9
    paddd           m5, m3
    punpcklwd       m0, m1, m6         ; b
    pmaddwd         m2, m0, m0         ; b * b
    punpckhwd       m1, m6
    pmaddwd         m3, m1, m1
    psubd           m4, m2             ; p
    psubd           m5, m3
    MULLD           m4, m9, m12        ; p * s
    MULLD           m5, m9, m12
    pmaddwd         m0, m10            ; b * 455
    pmaddwd         m1, m10
    paddusw         m4, m10
    paddusw         m5, m10
    psrld           m4, 20             ; min(z, 255)
    psrld           m5, 20
    GATHER_X_BY_X   m3, m4, m5, r0, dstm
    punpcklwd       m4, m3, m3
    punpckhwd       m5, m3, m3
    MULLD           m0, m4, m12
    MULLD           m1, m5, m12
%if ARCH_X86_32
    pxor            m6, m6
%endif
    paddd           m0, m11            ; x * b * 455 + (1 << 11) + (1 << 15)
    paddd           m1, m11
    mova [t4+wq*2+400*2+ 4], m3
    psrld           m0, 12
    psrld           m1, 12
    mova [t3+wq*4+400*4+ 8], m0
    mova [t3+wq*4+400*4+24], m1
    add             wq, 8
    jl .v1_loop
    mov            r10, t2
    mov             t2, t1
    mov             t1, r10
    ret
.prep_n: ; initial neighbor setup
    movif64         wq, r4
    movif32         wd, w1m
.prep_n_loop:
    movu            m0, [t4+wq*2+400*0+ 4]
    movu            m1, [t3+wq*4+400*0+ 8]
    movu            m2, [t3+wq*4+400*0+24]
    movu            m3, [t4+wq*2+400*0+ 2]
    movu            m4, [t3+wq*4+400*0+ 4]
    movu            m5, [t3+wq*4+400*0+20]
    paddw           m0, [t4+wq*2+400*0+ 0]
    paddd           m1, [t3+wq*4+400*0+ 0]
    paddd           m2, [t3+wq*4+400*0+16]
    paddw           m3, m0
    paddd           m4, m1
    paddd           m5, m2
    psllw           m3, 2                ; a[-1] 444
    pslld           m4, 2                ; b[-1] 444
    pslld           m5, 2
    psubw           m3, m0               ; a[-1] 343
    psubd           m4, m1               ; b[-1] 343
    psubd           m5, m2
    mova [t4+wq*2+400*4], m3
    mova [t3+wq*4+400*8+ 0], m4
    mova [t3+wq*4+400*8+16], m5
    movu            m0, [t4+wq*2+400*2+ 4]
    movu            m1, [t3+wq*4+400*4+ 8]
    movu            m2, [t3+wq*4+400*4+24]
    movu            m3, [t4+wq*2+400*2+ 2]
    movu            m4, [t3+wq*4+400*4+ 4]
    movu            m5, [t3+wq*4+400*4+20]
    paddw           m0, [t4+wq*2+400*2+ 0]
    paddd           m1, [t3+wq*4+400*4+ 0]
    paddd           m2, [t3+wq*4+400*4+16]
    paddw           m3, m0
    paddd           m4, m1
    paddd           m5, m2
    psllw           m3, 2                 ; a[ 0] 444
    pslld           m4, 2                 ; b[ 0] 444
    pslld           m5, 2
    mova [t4+wq*2+400* 6], m3
    mova [t3+wq*4+400*12+ 0], m4
    mova [t3+wq*4+400*12+16], m5
    psubw           m3, m0                ; a[ 0] 343
    psubd           m4, m1                ; b[ 0] 343
    psubd           m5, m2
    mova [t4+wq*2+400* 8], m3
    mova [t3+wq*4+400*16+ 0], m4
    mova [t3+wq*4+400*16+16], m5
    add             wq, 8
    jl .prep_n_loop
    ret
ALIGN function_align
.n0: ; neighbor + output (even rows)
    movif64         wq, r4
    movif32         wd, w1m
.n0_loop:
    movu            m3, [t4+wq*2+400*0+4]
    movu            m1, [t4+wq*2+400*0+2]
    paddw           m3, [t4+wq*2+400*0+0]
    paddw           m1, m3
    psllw           m1, 2                ; a[ 1] 444
    psubw           m2, m1, m3           ; a[ 1] 343
    paddw           m3, m2, [t4+wq*2+400*4]
    paddw           m3, [t4+wq*2+400*6]
    mova [t4+wq*2+400*4], m2
    mova [t4+wq*2+400*6], m1
    movu            m4, [t3+wq*4+400*0+8]
    movu            m1, [t3+wq*4+400*0+4]
    paddd           m4, [t3+wq*4+400*0+0]
    paddd           m1, m4
    pslld           m1, 2                ; b[ 1] 444
    psubd           m2, m1, m4           ; b[ 1] 343
    paddd           m4, m2, [t3+wq*4+400* 8+ 0]
    paddd           m4, [t3+wq*4+400*12+ 0]
    mova [t3+wq*4+400* 8+ 0], m2
    mova [t3+wq*4+400*12+ 0], m1
    movu            m5, [t3+wq*4+400*0+24]
    movu            m1, [t3+wq*4+400*0+20]
    paddd           m5, [t3+wq*4+400*0+16]
    paddd           m1, m5
    pslld           m1, 2
    psubd           m2, m1, m5
    paddd           m5, m2, [t3+wq*4+400* 8+16]
    paddd           m5, [t3+wq*4+400*12+16]
    mova [t3+wq*4+400* 8+16], m2
    mova [t3+wq*4+400*12+16], m1
    movq            m0, [dstq+wq]
    punpcklbw       m0, m6
    punpcklwd       m1, m0, m6
    punpcklwd       m2, m3, m6
    pmaddwd         m2, m1               ; a * src
    punpckhwd       m1, m0, m6
    punpckhwd       m3, m6
    pmaddwd         m3, m1
    psubd           m4, m2               ; b - a * src + (1 << 8)
    psubd           m5, m3
    psrad           m4, 9
    psrad           m5, 9
    packssdw        m4, m5
    pmulhrsw        m4, m7
    paddw           m0, m4
    packuswb        m0, m0
    movq     [dstq+wq], m0
    add             wq, 8
    jl .n0_loop
    add           dstq, stridemp
    ret
ALIGN function_align
.n1: ; neighbor + output (odd rows)
    movif64         wq, r4
    movif32         wd, w1m
.n1_loop:
    movu            m3, [t4+wq*2+400*2+4]
    movu            m1, [t4+wq*2+400*2+2]
    paddw           m3, [t4+wq*2+400*2+0]
    paddw           m1, m3
    psllw           m1, 2                ; a[ 1] 444
    psubw           m2, m1, m3           ; a[ 1] 343
    paddw           m3, m2, [t4+wq*2+400*6]
    paddw           m3, [t4+wq*2+400*8]
    mova [t4+wq*2+400*6], m1
    mova [t4+wq*2+400*8], m2
    movu            m4, [t3+wq*4+400*4+8]
    movu            m1, [t3+wq*4+400*4+4]
    paddd           m4, [t3+wq*4+400*4+0]
    paddd           m1, m4
    pslld           m1, 2                ; b[ 1] 444
    psubd           m2, m1, m4           ; b[ 1] 343
    paddd           m4, m2, [t3+wq*4+400*12+ 0]
    paddd           m4, [t3+wq*4+400*16+ 0]
    mova [t3+wq*4+400*12+ 0], m1
    mova [t3+wq*4+400*16+ 0], m2
    movu            m5, [t3+wq*4+400*4+24]
    movu            m1, [t3+wq*4+400*4+20]
    paddd           m5, [t3+wq*4+400*4+16]
    paddd           m1, m5
    pslld           m1, 2
    psubd           m2, m1, m5
    paddd           m5, m2, [t3+wq*4+400*12+16]
    paddd           m5, [t3+wq*4+400*16+16]
    mova [t3+wq*4+400*12+16], m1
    mova [t3+wq*4+400*16+16], m2
    movq            m0, [dstq+wq]
    punpcklbw       m0, m6
    punpcklwd       m1, m0, m6
    punpcklwd       m2, m3, m6
    pmaddwd         m2, m1               ; a * src
    punpckhwd       m1, m0, m6
    punpckhwd       m3, m6
    pmaddwd         m3, m1
    psubd           m4, m2               ; b - a * src + (1 << 8)
    psubd           m5, m3
    psrad           m4, 9
    psrad           m5, 9
    packssdw        m4, m5
    pmulhrsw        m4, m7
    paddw           m0, m4
    packuswb        m0, m0
    movq     [dstq+wq], m0
    add             wq, 8
    jl .n1_loop
    add           dstq, stridemp
    movif32       dstm, dstq
    ret

%if ARCH_X86_32
 %if STACK_ALIGNMENT < 16
  %assign extra_stack 10*16
 %else
  %assign extra_stack 8*16
 %endif
cglobal sgr_filter_mix_8bpc, 1, 7, 8, -400*66-48-extra_stack, \
                             dst, stride, left, lpf, w
 %if STACK_ALIGNMENT < 16
  %define dstm         dword [esp+calloff+16*8+4*0]
  %define stridemp     dword [esp+calloff+16*8+4*1]
  %define leftm        dword [esp+calloff+16*8+4*2]
  %define lpfm         dword [esp+calloff+16*8+4*3]
  %define w0m          dword [esp+calloff+16*8+4*4]
  %define hd           dword [esp+calloff+16*8+4*5]
  %define edgeb         byte [esp+calloff+16*8+4*6]
  %define edged        dword [esp+calloff+16*8+4*6]
  %define leftmp leftm
 %else
  %define w0m wm
  %define hd dword r5m
  %define edgeb  byte r7m
  %define edged dword r7m
 %endif
 %define hvsrcm dword [esp+calloff+4*0]
 %define w1m    dword [esp+calloff+4*1]
 %define t3m    dword [esp+calloff+4*2]
 %define t4m    dword [esp+calloff+4*3]
 %xdefine m8 m6
 %define  m9 [base+pd_0xffff]
 %define m10 [base+pd_34816]
 %define m11 [base+pd_0xf00801c7]
 %define m12 [base+pd_0xf00800a4]
 %define m13 [esp+calloff+16*4]
 %define m14 [esp+calloff+16*5]
 %define m15 [esp+calloff+16*6]
 %define  m6 [esp+calloff+16*7]
 %define base r6-$$
 %assign calloff 0
 %if STACK_ALIGNMENT < 16
    mov        strideq, [rstk+stack_offset+ 8]
    mov          leftq, [rstk+stack_offset+12]
    mov           lpfq, [rstk+stack_offset+16]
    mov             wd, [rstk+stack_offset+20]
    mov           dstm, dstq
    mov       stridemp, strideq
    mov          leftm, leftq
    mov             r1, [rstk+stack_offset+24]
    mov             r2, [rstk+stack_offset+32]
    mov           lpfm, lpfq
    mov             hd, r1
    mov          edged, r2
 %endif
%else
cglobal sgr_filter_mix_8bpc, 4, 15, 16, -400*66-40, dst, stride, left, lpf, \
                                                    w, h, edge, params
%endif
%if ARCH_X86_64 || STACK_ALIGNMENT >= 16
    mov             wd, wm
%endif
%if ARCH_X86_64
    mov        paramsq, r6mp
    lea            r13, [sgr_x_by_x-0xf03]
    movifnidn       hd, hm
    mov          edged, r7m
    mova           m15, [paramsq]
    add           lpfq, wq
    mova            m9, [pd_0xffff]
    lea             t1, [rsp+wq*2+44]
    mova           m10, [pd_34816]
    add           dstq, wq
    lea             t3, [rsp+wq*4+400*24+40]
    mova           m11, [pd_0xf00801c7]
    lea             t4, [rsp+wq*2+400*52+40]
    mova           m12, [base+pd_0xf00800a4]
    neg             wq
    pshuflw        m13, m15, q0000
    pshuflw        m14, m15, q2222
    pshufhw        m15, m15, q1010
    punpcklqdq     m13, m13 ; s0
    punpcklqdq     m14, m14 ; s1
    punpckhqdq     m15, m15 ; w0 w1
    pxor            m6, m6
    psllw          m15, 2
 DEFINE_ARGS dst, stride, left, lpf, _, h, edge, _, _, _, w
 %define lpfm [rsp]
%else
    mov             r1, [rstk+stack_offset+28] ; params
    LEA             r6, $$
    mova            m2, [r1]
    add           lpfm, wq
    lea             t1, [rsp+extra_stack+wq*2+52]
    add           dstq, wq
    lea             t3, [rsp+extra_stack+wq*4+400*24+48]
    mov           dstm, dstq
    lea             t4, [rsp+extra_stack+wq*2+400*52+48]
    mov            t3m, t3
    mov            t4m, t4
    neg             wq
    pshuflw         m0, m2, q0000
    pshuflw         m1, m2, q2222
    pshufhw         m2, m2, q1010
    punpcklqdq      m0, m0 ; s0
    punpcklqdq      m1, m1 ; s1
    punpckhqdq      m2, m2 ; w0 w1
    mov            w1m, wd
    pxor            m3, m3
    psllw           m2, 2
    mova           m13, m0
    mova           m14, m1
    sub             wd, 2
    mova           m15, m2
    mova            m6, m3
    mov           lpfq, lpfm
    mov            w0m, wd
 %define strideq r5
%endif
    test         edgeb, 4 ; LR_HAVE_TOP
    jz .no_top
    call .h_top
    add           lpfq, stridemp
    mov             t2, t1
%if ARCH_X86_64
    call mangle(private_prefix %+ _sgr_filter_5x5_8bpc_ssse3).top_fixup
%else
    mov             wq, w0m
    call mangle(private_prefix %+ _sgr_filter_5x5_8bpc_ssse3).top_fixup_loop
%endif
    add             t1, 400*12
    call .h_top
    movif32    strideq, stridemp
    lea            r10, [lpfq+strideq*4]
    mov           lpfq, dstq
    add            r10, strideq
    mov           lpfm, r10 ; below
    movif32         t4, t4m
    call .hv0
.main:
    dec             hd
    jz .height1
    movif32       lpfq, hvsrcm
    add           lpfq, stridemp
    call .hv1
    call .prep_n
    sub             hd, 2
    jl .extend_bottom
.main_loop:
    movif32       lpfq, hvsrcm
    add           lpfq, stridemp
    call .hv0
%if ARCH_X86_64
    test            hd, hd
%else
    mov             r4, hd
    test            r4, r4
%endif
    jz .odd_height
    movif32       lpfq, hvsrcm
    add           lpfq, stridemp
    call .hv1
    call .n0
    call .n1
    sub             hd, 2
    jge .main_loop
    test         edgeb, 8 ; LR_HAVE_BOTTOM
    jz .extend_bottom
    mov           lpfq, lpfm
    call .hv0_bottom
    movif32       lpfq, hvsrcm
    add           lpfq, stridemp
    call .hv1_bottom
.end:
    call .n0
    call .n1
.end2:
    RET
.height1:
    call .v1
    call .prep_n
    jmp .odd_height_end
.odd_height:
    call .v1
    call .n0
    call .n1
.odd_height_end:
    call .v0
    call .v1
    call .n0
    jmp .end2
.extend_bottom:
    call .v0
    call .v1
    jmp .end
.no_top:
    movif32    strideq, stridemp
    lea            r10, [lpfq+strideq*4]
    mov           lpfq, dstq
    lea            r10, [r10+strideq*2]
    mov           lpfm, r10
    call .h
%if ARCH_X86_64
    lea             wq, [r4-2]
%else
    mov             wq, w0m
    mov         hvsrcm, lpfq
%endif
    lea             t2, [t1+400*12]
.top_fixup_loop:
    mova            m0, [t1+wq*2+400* 0]
    mova            m1, [t1+wq*2+400* 2]
    mova            m2, [t1+wq*2+400* 4]
    paddw           m0, m0
    mova            m3, [t1+wq*2+400* 6]
    paddd           m1, m1
    mova            m4, [t1+wq*2+400* 8]
    paddd           m2, m2
    mova            m5, [t1+wq*2+400*10]
    mova [t2+wq*2+400* 0], m0
    mova [t2+wq*2+400* 2], m1
    mova [t2+wq*2+400* 4], m2
    mova [t2+wq*2+400* 6], m3
    mova [t2+wq*2+400* 8], m4
    mova [t2+wq*2+400*10], m5
    add             wq, 8
    jl .top_fixup_loop
    movif32         t3, t3m
    movif32         t4, t4m
    call .v0
    jmp .main
.extend_right:
%assign stack_offset stack_offset+8
%assign calloff 8
%if ARCH_X86_64
    SWAP            m8, m6
%endif
    movd            m1, wd
    movd            m3, [lpfq-1]
    pshufb          m1, m8
    pshufb          m3, m8
    psubb           m2, [base+pb_1], m1
    pcmpgtb         m2, [base+pb_0to15]
    pand            m5, m2
    pandn           m2, m3
    por             m5, m2
%if ARCH_X86_64
    SWAP            m6, m8
%endif
    ret
%assign stack_offset stack_offset-4
%assign calloff 4
.h: ; horizontal boxsum
%if ARCH_X86_64
    lea             wq, [r4-2]
%else
 %define leftq r4
%endif
    test         edgeb, 1 ; LR_HAVE_LEFT
    jz .h_extend_left
    movif32      leftq, leftm
    movddup         m4, [leftq-4]
    movif32         wq, w0m
    mova            m5, [lpfq+wq+2]
    add         leftmp, 4
    palignr         m5, m4, 13
    jmp .h_main
.h_extend_left:
    movif32         wq, w0m
    mova            m5, [lpfq+wq+2]
    pshufb          m5, [base+sgr_lshuf5]
    jmp .h_main
.h_top:
%if ARCH_X86_64
    lea             wq, [r4-2]
%endif
    test         edgeb, 1 ; LR_HAVE_LEFT
    jz .h_extend_left
    movif32         wq, w0m
.h_loop:
    movu            m5, [lpfq+wq-1]
.h_main:
    test         edgeb, 2 ; LR_HAVE_RIGHT
%if ARCH_X86_32
    pxor            m8, m8
%else
    SWAP            m8, m6
%endif
    jnz .h_have_right
    cmp             wd, -10
    jl .h_have_right
    call .extend_right
.h_have_right:
    punpcklbw       m4, m5, m8
    punpckhbw       m5, m8
    palignr         m3, m5, m4, 2
    palignr         m0, m5, m4, 4
    paddw           m1, m3, m0
    punpcklwd       m2, m3, m0
    pmaddwd         m2, m2
    punpckhwd       m3, m0
    pmaddwd         m3, m3
    palignr         m0, m5, m4, 6
    paddw           m1, m0             ; sum3
    punpcklwd       m7, m0, m8
    pmaddwd         m7, m7
    punpckhwd       m0, m8
    pmaddwd         m0, m0
%if ARCH_X86_64
    SWAP            m6, m8
%endif
    paddd           m2, m7             ; sumsq3
    palignr         m5, m4, 8
    punpcklwd       m7, m5, m4
    paddw           m8, m4, m5
    pmaddwd         m7, m7
    punpckhwd       m5, m4
    pmaddwd         m5, m5
    paddd           m3, m0
    mova [t1+wq*2+400* 6], m1
    mova [t1+wq*2+400* 8], m2
    mova [t1+wq*2+400*10], m3
    paddw           m8, m1             ; sum5
    paddd           m7, m2             ; sumsq5
    paddd           m5, m3
    mova [t1+wq*2+400* 0], m8
    mova [t1+wq*2+400* 2], m7
    mova [t1+wq*2+400* 4], m5
    add             wq, 8
    jl .h_loop
    ret
ALIGN function_align
.hv0: ; horizontal boxsum + vertical boxsum + ab3 (even rows)
%if ARCH_X86_64
    lea             wq, [r4-2]
%else
    mov         hvsrcm, lpfq
%endif
    test         edgeb, 1 ; LR_HAVE_LEFT
    jz .hv0_extend_left
    movif32      leftq, leftm
    movddup         m4, [leftq-4]
    movif32         wq, w0m
    mova            m5, [lpfq+wq+2]
    add         leftmp, 4
    palignr         m5, m4, 13
    jmp .hv0_main
.hv0_extend_left:
    movif32         wq, w0m
    mova            m5, [lpfq+wq+2]
    pshufb          m5, [base+sgr_lshuf5]
    jmp .hv0_main
.hv0_bottom:
%if ARCH_X86_64
    lea             wq, [r4-2]
%else
    mov         hvsrcm, lpfq
%endif
    test         edgeb, 1 ; LR_HAVE_LEFT
    jz .hv0_extend_left
    movif32         wq, w0m
%if ARCH_X86_32
    jmp .hv0_loop_start
%endif
.hv0_loop:
    movif32       lpfq, hvsrcm
.hv0_loop_start:
    movu            m5, [lpfq+wq-1]
.hv0_main:
    test         edgeb, 2 ; LR_HAVE_RIGHT
%if ARCH_X86_32
    pxor            m8, m8
%else
    SWAP            m8, m6
%endif
    jnz .hv0_have_right
    cmp             wd, -10
    jl .hv0_have_right
    call .extend_right
.hv0_have_right:
    punpcklbw       m4, m5, m8
    punpckhbw       m5, m8
    palignr         m3, m5, m4, 2
    palignr         m0, m5, m4, 4
    movif32         t3, t3m
    paddw           m1, m3, m0
    punpcklwd       m2, m3, m0
    pmaddwd         m2, m2
    punpckhwd       m3, m0
    pmaddwd         m3, m3
    palignr         m0, m5, m4, 6
    paddw           m1, m0             ; h sum3
    punpcklwd       m7, m0, m8
    pmaddwd         m7, m7
    punpckhwd       m0, m8
%if ARCH_X86_64
    SWAP            m6, m8
%endif
    pmaddwd         m0, m0
    paddd           m2, m7             ; h sumsq3
    palignr         m5, m4, 8
    punpcklwd       m7, m5, m4
    paddw           m8, m4, m5
    pmaddwd         m7, m7
    punpckhwd       m5, m4
    pmaddwd         m5, m5
    paddd           m3, m0
    paddw           m8, m1             ; h sum5
    paddd           m7, m2             ; h sumsq5
    paddd           m5, m3
    mova [t3+wq*4+400*8+ 8], m8
    mova [t3+wq*4+400*0+ 8], m7
    mova [t3+wq*4+400*0+24], m5
    paddw           m8, [t1+wq*2+400* 0]
    paddd           m7, [t1+wq*2+400* 2]
    paddd           m5, [t1+wq*2+400* 4]
    mova [t1+wq*2+400* 0], m8
    mova [t1+wq*2+400* 2], m7
    mova [t1+wq*2+400* 4], m5
    paddw           m0, m1, [t1+wq*2+400* 6]
    paddd           m4, m2, [t1+wq*2+400* 8]
    paddd           m5, m3, [t1+wq*2+400*10]
    mova [t1+wq*2+400* 6], m1
    mova [t1+wq*2+400* 8], m2
    mova [t1+wq*2+400*10], m3
    paddw           m1, m0, [t2+wq*2+400* 6]
    paddd           m2, m4, [t2+wq*2+400* 8]
    paddd           m3, m5, [t2+wq*2+400*10]
    mova [t2+wq*2+400* 6], m0
    mova [t2+wq*2+400* 8], m4
    mova [t2+wq*2+400*10], m5
%if ARCH_X86_32
    pxor            m7, m7
%else
    SWAP            m7, m6
%endif
    pslld           m4, m2, 3
    pslld           m5, m3, 3
    paddd           m4, m2             ; a3 * 9
    paddd           m5, m3
    punpcklwd       m0, m1, m7         ; b3
    pmaddwd         m2, m0, m0
    punpckhwd       m1, m7
    pmaddwd         m3, m1, m1
%if ARCH_X86_64
    SWAP            m7, m6
%endif
    psubd           m4, m2             ; p3
    psubd           m5, m3
    MULLD           m4, m14, m7        ; p3 * s1
    MULLD           m5, m14, m7
    pmaddwd         m0, m11            ; b3 * 455
    pmaddwd         m1, m11
    paddusw         m4, m11
    paddusw         m5, m11
    psrld           m4, 20             ; min(z3, 255)
    psrld           m5, 20
    GATHER_X_BY_X   m3, m4, m5, r0, dstm
    punpcklwd       m4, m3, m3
    punpckhwd       m5, m3, m3
    MULLD           m0, m4, m7
    MULLD           m1, m5, m7
    paddd           m0, m10            ; x3 * b3 * 455 + (1 << 11) + (1 << 15)
    paddd           m1, m10
    mova [t4+wq*2+400*2+ 4], m3
    psrld           m0, 12
    psrld           m1, 12
    mova [t3+wq*4+400*4+ 8], m0
    mova [t3+wq*4+400*4+24], m1
    add             wq, 8
    jl .hv0_loop
    ret
ALIGN function_align
.hv1: ; horizontal boxsums + vertical boxsums + ab (odd rows)
%if ARCH_X86_64
    lea             wq, [r4-2]
%else
    mov         hvsrcm, lpfq
%endif
    test         edgeb, 1 ; LR_HAVE_LEFT
    jz .hv1_extend_left
    movif32      leftq, leftm
    movddup         m4, [leftq-4]
    movif32         wq, w0m
    mova            m5, [lpfq+wq+2]
    add         leftmp, 4
    palignr         m5, m4, 13
    jmp .hv1_main
.hv1_extend_left:
    movif32         wq, w0m
    mova            m5, [lpfq+wq+2]
    pshufb          m5, [base+sgr_lshuf5]
    jmp .hv1_main
.hv1_bottom:
%if ARCH_X86_64
    lea             wq, [r4-2]
%else
    mov         hvsrcm, lpfq
%endif
    test         edgeb, 1 ; LR_HAVE_LEFT
    jz .hv1_extend_left
    movif32         wq, w0m
%if ARCH_X86_32
    jmp .hv1_loop_start
%endif
.hv1_loop:
    movif32       lpfq, hvsrcm
.hv1_loop_start:
    movu            m5, [lpfq+wq-1]
.hv1_main:
    test         edgeb, 2 ; LR_HAVE_RIGHT
%if ARCH_X86_32
    pxor            m8, m8
%else
    SWAP            m8, m6
%endif
    jnz .hv1_have_right
    cmp             wd, -10
    jl .hv1_have_right
    call .extend_right
.hv1_have_right:
    punpcklbw       m4, m5, m8
    punpckhbw       m5, m8
    palignr         m7, m5, m4, 2
    palignr         m3, m5, m4, 4
    paddw           m2, m7, m3
    punpcklwd       m0, m7, m3
    pmaddwd         m0, m0
    punpckhwd       m7, m3
    pmaddwd         m7, m7
    palignr         m3, m5, m4, 6
    paddw           m2, m3             ; h sum3
    punpcklwd       m1, m3, m8
    pmaddwd         m1, m1
    punpckhwd       m3, m8
%if ARCH_X86_64
    SWAP            m6, m8
%endif
    pmaddwd         m3, m3
    paddd           m0, m1             ; h sumsq3
    palignr         m5, m4, 8
    punpckhwd       m1, m4, m5
    paddw           m8, m4, m5
    pmaddwd         m1, m1
    punpcklwd       m4, m5
    pmaddwd         m4, m4
    paddd           m7, m3
    paddw           m5, m2, [t2+wq*2+400* 6]
    mova [t2+wq*2+400* 6], m2
    paddw           m8, m2             ; h sum5
    paddd           m2, m0, [t2+wq*2+400* 8]
    paddd           m3, m7, [t2+wq*2+400*10]
    mova [t2+wq*2+400* 8], m0
    mova [t2+wq*2+400*10], m7
    paddd           m4, m0             ; h sumsq5
    paddd           m1, m7
    pslld           m0, m2, 3
    pslld           m7, m3, 3
    paddd           m2, m0             ; a3 * 9
    paddd           m3, m7
%if ARCH_X86_32
    mova      [esp+20], m8
    pxor            m8, m8
%else
    SWAP            m8, m6
%endif
    punpcklwd       m0, m5, m8         ; b3
    pmaddwd         m7, m0, m0
    punpckhwd       m5, m8
    pmaddwd         m8, m5, m5
    psubd           m2, m7             ; p3
    psubd           m3, m8
    MULLD           m2, m14, m8        ; p3 * s1
    MULLD           m3, m14, m8
    pmaddwd         m0, m11            ; b3 * 455
    pmaddwd         m5, m11
    paddusw         m2, m11
    paddusw         m3, m11
    psrld           m2, 20             ; min(z3, 255)
    movif32         t3, t3m
    psrld           m3, 20
    GATHER_X_BY_X   m8, m2, m3, r0, dstm
    punpcklwd       m2, m8, m8
    punpckhwd       m3, m8, m8
    MULLD           m0, m2, m7
    MULLD           m5, m3, m7
    paddd           m0, m10            ; x3 * b3 * 455 + (1 << 11) + (1 << 15)
    paddd           m5, m10
    psrld           m0, 12
    psrld           m5, 12
    mova [t4+wq*2+400*4+ 4], m8
    mova [t3+wq*4+400*8+ 8], m0
    mova [t3+wq*4+400*8+24], m5
%if ARCH_X86_32
    mova            m8, [esp+20]
%else
    SWAP            m6, m8
    pxor            m6, m6
%endif
    paddw           m5, m8, [t2+wq*2+400*0]
    paddd           m2, m4, [t2+wq*2+400*2]
    paddd           m3, m1, [t2+wq*2+400*4]
    paddw           m5, [t1+wq*2+400*0]
    paddd           m2, [t1+wq*2+400*2]
    paddd           m3, [t1+wq*2+400*4]
    mova [t2+wq*2+400*0], m8
    pslld           m0, m2, 4
    mova [t2+wq*2+400*2], m4
    pslld           m8, m3, 4
    mova [t2+wq*2+400*4], m1
    pslld           m4, m2, 3
    paddd           m2, m0
    pslld           m7, m3, 3
    paddd           m3, m8
    paddd           m2, m4             ; a5 * 25
    paddd           m3, m7
%if ARCH_X86_32
    pxor            m7, m7
%else
    SWAP            m7, m6
%endif
    punpcklwd       m0, m5, m7         ; b5
    pmaddwd         m4, m0, m0
    punpckhwd       m5, m7
    pmaddwd         m1, m5, m5
%if ARCH_X86_64
    SWAP            m7, m6
%endif
    psubd           m2, m4             ; p5
    psubd           m3, m1
    MULLD           m2, m13, m7        ; p5 * s0
    MULLD           m3, m13, m7
    pmaddwd         m0, m12            ; b5 * 164
    pmaddwd         m5, m12
    paddusw         m2, m12
    paddusw         m3, m12
    psrld           m2, 20             ; min(z5, 255)
    psrld           m3, 20
    GATHER_X_BY_X   m1, m2, m3, r0, dstm
    punpcklwd       m2, m1, m1
    punpckhwd       m3, m1, m1
    MULLD           m0, m2, m7
    MULLD           m5, m3, m7
    paddd           m0, m10            ; x5 * b5 * 164 + (1 << 11) + (1 << 15)
    paddd           m5, m10
    mova   [t4+wq*2+4], m1
    psrld           m0, 12
    psrld           m5, 12
    mova  [t3+wq*4+ 8], m0
    mova  [t3+wq*4+24], m5
    add             wq, 8
    jl .hv1_loop
    mov            r10, t2
    mov             t2, t1
    mov             t1, r10
    ret
.v0: ; vertical boxsums + ab3 (even rows)
%if ARCH_X86_64
    lea             wq, [r4-2]
%else
    mov             wd, w0m
%endif
.v0_loop:
    mova            m0, [t1+wq*2+400* 6]
    mova            m4, [t1+wq*2+400* 8]
    mova            m5, [t1+wq*2+400*10]
    paddw           m0, m0
    paddd           m4, m4
    paddd           m5, m5
    paddw           m1, m0, [t2+wq*2+400* 6]
    paddd           m2, m4, [t2+wq*2+400* 8]
    paddd           m3, m5, [t2+wq*2+400*10]
    mova [t2+wq*2+400* 6], m0
    mova [t2+wq*2+400* 8], m4
    mova [t2+wq*2+400*10], m5
%if ARCH_X86_32
    pxor            m7, m7
%else
    SWAP            m7, m6
%endif
    pslld           m4, m2, 3
    pslld           m5, m3, 3
    paddd           m4, m2             ; a3 * 9
    paddd           m5, m3
    punpcklwd       m0, m1, m7         ; b3
    pmaddwd         m2, m0, m0
    punpckhwd       m1, m7
    pmaddwd         m3, m1, m1
    psubd           m4, m2             ; p3
    psubd           m5, m3
%if ARCH_X86_64
    SWAP            m7, m6
%endif
    MULLD           m4, m14, m7        ; p3 * s1
    MULLD           m5, m14, m7
    pmaddwd         m0, m11            ; b3 * 455
    pmaddwd         m1, m11
    paddusw         m4, m11
    paddusw         m5, m11
    psrld           m4, 20             ; min(z3, 255)
    psrld           m5, 20
    GATHER_X_BY_X   m3, m4, m5, r0, dstm
    punpcklwd       m4, m3, m3
    punpckhwd       m5, m3, m3
    MULLD           m0, m4, m7
    MULLD           m1, m5, m7
    paddd           m0, m10            ; x3 * b3 * 455 + (1 << 11) + (1 << 15)
    paddd           m1, m10
    mova [t4+wq*2+400*2+4], m3
    psrld           m0, 12
    psrld           m1, 12
    mova            m3, [t1+wq*2+400*0]
    mova            m4, [t1+wq*2+400*2]
    mova            m5, [t1+wq*2+400*4]
    mova [t3+wq*4+400*8+ 8], m3
    mova [t3+wq*4+400*0+ 8], m4
    mova [t3+wq*4+400*0+24], m5
    paddw           m3, m3 ; cc5
    paddd           m4, m4
    paddd           m5, m5
    mova [t1+wq*2+400*0], m3
    mova [t1+wq*2+400*2], m4
    mova [t1+wq*2+400*4], m5
    mova [t3+wq*4+400*4+ 8], m0
    mova [t3+wq*4+400*4+24], m1
    add             wq, 8
    jl .v0_loop
    ret
.v1: ; vertical boxsums + ab (odd rows)
%if ARCH_X86_64
    lea             wq, [r4-2]
%else
    mov             wd, w0m
%endif
.v1_loop:
    mova            m4, [t1+wq*2+400* 6]
    mova            m5, [t1+wq*2+400* 8]
    mova            m7, [t1+wq*2+400*10]
    paddw           m1, m4, [t2+wq*2+400* 6]
    paddd           m2, m5, [t2+wq*2+400* 8]
    paddd           m3, m7, [t2+wq*2+400*10]
    mova [t2+wq*2+400* 6], m4
    mova [t2+wq*2+400* 8], m5
    mova [t2+wq*2+400*10], m7
%if ARCH_X86_32
    pxor            m7, m7
%else
    SWAP            m7, m6
%endif
    pslld           m4, m2, 3
    pslld           m5, m3, 3
    paddd           m4, m2             ; ((a3 + 8) >> 4) * 9
    paddd           m5, m3
    punpcklwd       m0, m1, m7         ; b3
    pmaddwd         m2, m0, m0
    punpckhwd       m1, m7
    pmaddwd         m3, m1, m1
    psubd           m4, m2             ; p3
    psubd           m5, m3
%if ARCH_X86_64
    SWAP            m7, m6
%endif
    MULLD           m4, m14, m7        ; p3 * s1
    MULLD           m5, m14, m7
    pmaddwd         m0, m11            ; b3 * 455
    pmaddwd         m1, m11
    paddusw         m4, m11
    paddusw         m5, m11
    psrld           m4, 20             ; min(z3, 255)
    psrld           m5, 20
    GATHER_X_BY_X   m3, m4, m5, r0, dstm
    punpcklwd       m4, m3, m3
    punpckhwd       m5, m3, m3
    MULLD           m0, m4, m7
    MULLD           m1, m5, m7
    paddd           m0, m10            ; x3 * b3 * 455 + (1 << 11) + (1 << 15)
    paddd           m1, m10
    mova [t4+wq*2+400*4+4], m3
    psrld           m0, 12
    psrld           m8, m1, 12
    mova            m4, [t3+wq*4+400*8+ 8]
    mova            m5, [t3+wq*4+400*0+ 8]
    mova            m7, [t3+wq*4+400*0+24]
    paddw           m1, m4, [t2+wq*2+400*0]
    paddd           m2, m5, [t2+wq*2+400*2]
    paddd           m3, m7, [t2+wq*2+400*4]
    paddw           m1, [t1+wq*2+400*0]
    paddd           m2, [t1+wq*2+400*2]
    paddd           m3, [t1+wq*2+400*4]
    mova [t2+wq*2+400*0], m4
    mova [t2+wq*2+400*2], m5
    mova [t2+wq*2+400*4], m7
    pslld           m4, m2, 4
    mova [t3+wq*4+400*8+ 8], m0
    pslld           m5, m3, 4
    mova [t3+wq*4+400*8+24], m8
    pslld           m7, m2, 3
    paddd           m2, m4
    pslld           m8, m3, 3
    paddd           m3, m5
    paddd           m2, m7             ; a5 * 25
    paddd           m3, m8
%if ARCH_X86_32
    pxor            m7, m7
%else
    SWAP            m7, m6
%endif
    punpcklwd       m0, m1, m7         ; b5
    pmaddwd         m4, m0, m0
    punpckhwd       m1, m7
    pmaddwd         m5, m1, m1
    psubd           m2, m4             ; p5
    psubd           m3, m5
%if ARCH_X86_64
    SWAP            m7, m6
%endif
    MULLD           m2, m13, m7        ; p5 * s0
    MULLD           m3, m13, m7
    pmaddwd         m0, m12            ; b5 * 164
    pmaddwd         m1, m12
    paddusw         m2, m12
    paddusw         m3, m12
    psrld           m2, 20             ; min(z5, 255)
    psrld           m3, 20
    GATHER_X_BY_X   m4, m2, m3, r0, dstm
    punpcklwd       m2, m4, m4
    punpckhwd       m3, m4, m4
    MULLD           m0, m2, m7
    MULLD           m1, m3, m7
    paddd           m0, m10            ; x5 * b5 * 164 + (1 << 11) + (1 << 15)
    paddd           m1, m10
    mova   [t4+wq*2+4], m4
    psrld           m0, 12
    psrld           m1, 12
    mova  [t3+wq*4+ 8], m0
    mova  [t3+wq*4+24], m1
    add             wq, 8
    jl .v1_loop
    mov            r10, t2
    mov             t2, t1
    mov             t1, r10
    ret
.prep_n: ; initial neighbor setup
    movif64         wq, r4
    movif32         wd, w1m
.prep_n_loop:
    movu            m0, [t4+wq*2+400*0+ 2]
    movu            m1, [t3+wq*4+400*0+ 4]
    movu            m2, [t3+wq*4+400*0+20]
    movu            m7, [t4+wq*2+400*0+ 4]
    movu            m8, [t3+wq*4+400*0+ 8]
    paddw           m3, m0, [t4+wq*2+400*0+ 0]
    paddd           m4, m1, [t3+wq*4+400*0+ 0]
    paddd           m5, m2, [t3+wq*4+400*0+16]
    paddw           m3, m7
    paddd           m4, m8
    movu            m7, [t3+wq*4+400*0+24]
    paddw           m0, m3
    paddd           m1, m4
    psllw           m3, 2
    pslld           m4, 2
    paddd           m5, m7
    paddd           m2, m5
    pslld           m5, 2
    paddw           m0, m3               ; a5 565
    paddd           m1, m4               ; b5 565
    paddd           m2, m5
    mova [t4+wq*2+400* 6+ 0], m0
    mova [t3+wq*4+400*12+ 0], m1
    mova [t3+wq*4+400*12+16], m2
    movu            m0, [t4+wq*2+400*2+ 4]
    movu            m1, [t3+wq*4+400*4+ 8]
    movu            m2, [t3+wq*4+400*4+24]
    movu            m3, [t4+wq*2+400*2+ 2]
    movu            m4, [t3+wq*4+400*4+ 4]
    movu            m5, [t3+wq*4+400*4+20]
    paddw           m0, [t4+wq*2+400*2+ 0]
    paddd           m1, [t3+wq*4+400*4+ 0]
    paddd           m2, [t3+wq*4+400*4+16]
    paddw           m3, m0
    paddd           m4, m1
    paddd           m5, m2
    psllw           m3, 2                ; a3[-1] 444
    pslld           m4, 2                ; b3[-1] 444
    pslld           m5, 2
    psubw           m3, m0               ; a3[-1] 343
    psubd           m4, m1               ; b3[-1] 343
    psubd           m5, m2
    mova [t4+wq*2+400* 8+ 0], m3
    mova [t3+wq*4+400*16+ 0], m4
    mova [t3+wq*4+400*16+16], m5
    movu            m0, [t4+wq*2+400*4+ 4]
    movu            m1, [t3+wq*4+400*8+ 8]
    movu            m2, [t3+wq*4+400*8+24]
    movu            m3, [t4+wq*2+400*4+ 2]
    movu            m4, [t3+wq*4+400*8+ 4]
    movu            m5, [t3+wq*4+400*8+20]
    paddw           m0, [t4+wq*2+400*4+ 0]
    paddd           m1, [t3+wq*4+400*8+ 0]
    paddd           m2, [t3+wq*4+400*8+16]
    paddw           m3, m0
    paddd           m4, m1
    paddd           m5, m2
    psllw           m3, 2                 ; a3[ 0] 444
    pslld           m4, 2                 ; b3[ 0] 444
    pslld           m5, 2
    mova [t4+wq*2+400*10+ 0], m3
    mova [t3+wq*4+400*20+ 0], m4
    mova [t3+wq*4+400*20+16], m5
    psubw           m3, m0                ; a3[ 0] 343
    psubd           m4, m1                ; b3[ 0] 343
    psubd           m5, m2
    mova [t4+wq*2+400*12+ 0], m3
    mova [t3+wq*4+400*24+ 0], m4
    mova [t3+wq*4+400*24+16], m5
    add             wq, 8
    jl .prep_n_loop
    ret
ALIGN function_align
.n0: ; neighbor + output (even rows)
    movif64         wq, r4
    movif32         wd, w1m
.n0_loop:
    movu            m0, [t4+wq*2+ 4]
    movu            m2, [t4+wq*2+ 2]
    paddw           m0, [t4+wq*2+ 0]
    paddw           m0, m2
    paddw           m2, m0
    psllw           m0, 2
    paddw           m0, m2               ; a5
    movu            m4, [t3+wq*4+ 8]
    movu            m5, [t3+wq*4+24]
    movu            m1, [t3+wq*4+ 4]
    movu            m3, [t3+wq*4+20]
    paddd           m4, [t3+wq*4+ 0]
    paddd           m5, [t3+wq*4+16]
    paddd           m4, m1
    paddd           m5, m3
    paddd           m1, m4
    paddd           m3, m5
    pslld           m4, 2
    pslld           m5, 2
    paddd           m4, m1               ; b5
    paddd           m5, m3
    movu            m2, [t4+wq*2+400* 6]
    paddw           m2, m0
    mova [t4+wq*2+400* 6], m0
    paddd           m0, m4, [t3+wq*4+400*12+ 0]
    paddd           m1, m5, [t3+wq*4+400*12+16]
    mova [t3+wq*4+400*12+ 0], m4
    mova [t3+wq*4+400*12+16], m5
    mova [rsp+16+ARCH_X86_32*4], m1
    movu            m3, [t4+wq*2+400*2+4]
    movu            m5, [t4+wq*2+400*2+2]
    paddw           m3, [t4+wq*2+400*2+0]
    paddw           m5, m3
    psllw           m5, 2                ; a3[ 1] 444
    psubw           m4, m5, m3           ; a3[ 1] 343
    movu            m3, [t4+wq*2+400* 8]
    paddw           m3, [t4+wq*2+400*10]
    paddw           m3, m4
    mova [t4+wq*2+400* 8], m4
    mova [t4+wq*2+400*10], m5
    movu            m1, [t3+wq*4+400*4+ 8]
    movu            m5, [t3+wq*4+400*4+ 4]
    movu            m7, [t3+wq*4+400*4+24]
    movu            m8, [t3+wq*4+400*4+20]
    paddd           m1, [t3+wq*4+400*4+ 0]
    paddd           m7, [t3+wq*4+400*4+16]
    paddd           m5, m1
    paddd           m8, m7
    pslld           m5, 2                ; b3[ 1] 444
    pslld           m8, 2
    psubd           m4, m5, m1           ; b3[ 1] 343
%if ARCH_X86_32
    mova      [esp+52], m8
    psubd           m8, m7
%else
    psubd           m6, m8, m7
    SWAP            m8, m6
%endif
    paddd           m1, m4, [t3+wq*4+400*16+ 0]
    paddd           m7, m8, [t3+wq*4+400*16+16]
    paddd           m1, [t3+wq*4+400*20+ 0]
    paddd           m7, [t3+wq*4+400*20+16]
    mova [t3+wq*4+400*16+ 0], m4
    mova [t3+wq*4+400*16+16], m8
    mova [t3+wq*4+400*20+ 0], m5
%if ARCH_X86_32
    mova            m8, [esp+52]
%else
    SWAP            m8, m6
    pxor            m6, m6
%endif
    mova [t3+wq*4+400*20+16], m8
    mova [rsp+32+ARCH_X86_32*4], m7
    movq            m4, [dstq+wq]
    punpcklbw       m4, m6
    punpcklwd       m5, m4, m6
    punpcklwd       m7, m2, m6
    pmaddwd         m7, m5               ; a5 * src
    punpcklwd       m8, m3, m6
    pmaddwd         m8, m5               ; a3 * src
    punpckhwd       m5, m4, m6
    punpckhwd       m2, m6
    pmaddwd         m2, m5
    punpckhwd       m3, m6
    pmaddwd         m3, m5
    psubd           m0, m7               ; b5 - a5 * src + (1 << 8) - (src << 13)
    psubd           m1, m8               ; b3 - a3 * src + (1 << 8) - (src << 13)
    psrld           m0, 9
    pslld           m1, 7
    pand            m0, m9
    pandn           m8, m9, m1
    por             m0, m8
    mova            m1, [rsp+16+ARCH_X86_32*4]
    psubd           m1, m2
    mova            m2, [rsp+32+ARCH_X86_32*4]
    psubd           m2, m3
    mova            m3, [base+pd_4096]
    psrld           m1, 9
    pslld           m2, 7
    pand            m1, m9
    pandn           m5, m9, m2
    por             m1, m5
    pmaddwd         m0, m15
    pmaddwd         m1, m15
    paddd           m0, m3
    paddd           m1, m3
    psrad           m0, 13
    psrad           m1, 13
    packssdw        m0, m1
    paddw           m0, m4
    packuswb        m0, m0
    movq     [dstq+wq], m0
    add             wq, 8
    jl .n0_loop
    add           dstq, stridemp
    ret
ALIGN function_align
.n1: ; neighbor + output (odd rows)
    movif64         wq, r4
    movif32         wd, w1m
.n1_loop:
    movu            m3, [t4+wq*2+400*4+4]
    movu            m5, [t4+wq*2+400*4+2]
    paddw           m3, [t4+wq*2+400*4+0]
    paddw           m5, m3
    psllw           m5, 2                ; a3[ 1] 444
    psubw           m4, m5, m3           ; a3[ 1] 343
    paddw           m3, m4, [t4+wq*2+400*12]
    paddw           m3, [t4+wq*2+400*10]
    mova [t4+wq*2+400*10], m5
    mova [t4+wq*2+400*12], m4
    movu            m1, [t3+wq*4+400*8+ 8]
    movu            m5, [t3+wq*4+400*8+ 4]
    movu            m7, [t3+wq*4+400*8+24]
    movu            m8, [t3+wq*4+400*8+20]
    paddd           m1, [t3+wq*4+400*8+ 0]
    paddd           m7, [t3+wq*4+400*8+16]
    paddd           m5, m1
    paddd           m8, m7
    pslld           m5, 2                ; b3[ 1] 444
    pslld           m8, 2
    psubd           m4, m5, m1           ; b3[ 1] 343
    psubd           m0, m8, m7
    paddd           m1, m4, [t3+wq*4+400*24+ 0]
    paddd           m7, m0, [t3+wq*4+400*24+16]
    paddd           m1, [t3+wq*4+400*20+ 0]
    paddd           m7, [t3+wq*4+400*20+16]
    mova [t3+wq*4+400*20+ 0], m5
    mova [t3+wq*4+400*20+16], m8
    mova [t3+wq*4+400*24+ 0], m4
    mova [t3+wq*4+400*24+16], m0
    movq            m5, [dstq+wq]
    mova            m2, [t4+wq*2+400* 6]
    punpcklbw       m5, m6
    punpcklwd       m4, m5, m6
    punpcklwd       m8, m2, m6
    pmaddwd         m8, m4               ; a5 * src
    punpcklwd       m0, m3, m6
    pmaddwd         m0, m4               ; a3 * src
    punpckhwd       m4, m5, m6
    punpckhwd       m2, m6
    pmaddwd         m2, m4
    punpckhwd       m3, m6
    pmaddwd         m3, m4
    psubd           m1, m0               ; b3 - a3 * src + (1 << 8) - (src << 13)
    mova            m0, [t3+wq*4+400*12+ 0]
    psubd           m0, m8               ; b5 - a5 * src + (1 << 8) - (src << 13)
    mova            m4, [t3+wq*4+400*12+16]
    psubd           m4, m2
    psubd           m7, m3
    pslld           m1, 7
    psrld           m0, 8
    psrld           m4, 8
    pslld           m7, 7
    pandn           m3, m9, m1
    pand            m0, m9
    por             m0, m3
    pand            m4, m9
    pandn           m2, m9, m7
    por             m2, m4
    mova            m1, [base+pd_4096]
    pmaddwd         m0, m15
    pmaddwd         m2, m15
    paddd           m0, m1
    paddd           m2, m1
    psrad           m0, 13
    psrad           m2, 13
    packssdw        m0, m2
    paddw           m0, m5
    packuswb        m0, m0
    movq     [dstq+wq], m0
    add             wq, 8
    jl .n1_loop
    add           dstq, stridemp
    movif32       dstm, dstq
    ret