path: root/third_party/dav1d/src/x86/ipred16_sse.asm

; Copyright © 2021, VideoLAN and dav1d authors
; Copyright © 2021, Two Orioles, LLC
; All rights reserved.
;
; Redistribution and use in source and binary forms, with or without
; modification, are permitted provided that the following conditions are met:
;
; 1. Redistributions of source code must retain the above copyright notice, this
;    list of conditions and the following disclaimer.
;
; 2. Redistributions in binary form must reproduce the above copyright notice,
;    this list of conditions and the following disclaimer in the documentation
;    and/or other materials provided with the distribution.
;
; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
; ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
; WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
; ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
; (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
; ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
; SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

%include "config.asm"
%include "ext/x86/x86inc.asm"

SECTION_RODATA

filter_shuf:   db  6,  7,  8,  9, 10, 11, 12, 13, 14, 15,  4,  5,  2,  3, -1, -1
pal_pred_shuf: db  0,  2,  4,  6,  8, 10, 12, 14,  1,  3,  5,  7,  9, 11, 13, 15

pb_0_1:  times 4 db 0, 1
pb_2_3:  times 4 db 2, 3
pw_1:    times 4 dw 1
pw_2:    times 4 dw 2
pw_4:    times 4 dw 4
pw_512:  times 4 dw 512
pw_2048: times 4 dw 2048

%macro JMP_TABLE 3-*
    %xdefine %1_%2_table (%%table - 2*4)
    %xdefine %%base mangle(private_prefix %+ _%1_%2)
    %%table:
    %rep %0 - 2
        dd %%base %+ .%3 - (%%table - 2*4)
        %rotate 1
    %endrep
%endmacro

%define ipred_dc_splat_16bpc_ssse3_table (ipred_dc_16bpc_ssse3_table + 10*4)
%define ipred_dc_128_16bpc_ssse3_table   (ipred_dc_16bpc_ssse3_table + 15*4)
%define ipred_cfl_splat_16bpc_ssse3_table (ipred_cfl_16bpc_ssse3_table + 8*4)

JMP_TABLE ipred_dc_left_16bpc,    ssse3, h4, h8, h16, h32, h64
JMP_TABLE ipred_dc_16bpc,         ssse3, h4, h8, h16, h32, h64, w4, w8, w16, w32, w64, \
                                         s4-10*4, s8-10*4, s16-10*4, s32-10*4, s64-10*4, \
                                         s4-15*4, s8-15*4, s16c-15*4, s32c-15*4, s64-15*4
JMP_TABLE ipred_h_16bpc,          ssse3, w4, w8, w16, w32, w64
JMP_TABLE ipred_cfl_16bpc,        ssse3, h4, h8, h16, h32, w4, w8, w16, w32, \
                                         s4-8*4, s8-8*4, s16-8*4, s32-8*4
JMP_TABLE ipred_cfl_left_16bpc,   ssse3, h4, h8, h16, h32
JMP_TABLE ipred_cfl_ac_444_16bpc, ssse3, w4, w8, w16, w32
JMP_TABLE pal_pred_16bpc,         ssse3, w4, w8, w16, w32, w64

cextern smooth_weights_1d_16bpc
cextern smooth_weights_2d_16bpc
cextern filter_intra_taps

SECTION .text

INIT_XMM ssse3
cglobal ipred_dc_top_16bpc, 3, 7, 6, dst, stride, tl, w, h
    LEA                  r5, ipred_dc_left_16bpc_ssse3_table
    movd                 m4, wm
    tzcnt                wd, wm
    add                 tlq, 2
    movifnidn            hd, hm
    pxor                 m3, m3
    pavgw                m4, m3
    movd                 m5, wd
    movu                 m0, [tlq]
    movsxd               r6, [r5+wq*4]
    add                  r6, r5
    add                  r5, ipred_dc_128_16bpc_ssse3_table-ipred_dc_left_16bpc_ssse3_table
    movsxd               wq, [r5+wq*4]
    add                  wq, r5
    jmp                  r6

cglobal ipred_dc_left_16bpc, 3, 7, 6, dst, stride, tl, w, h, stride3
    LEA                  r5, ipred_dc_left_16bpc_ssse3_table
    mov                  hd, hm
    movd                 m4, hm
    tzcnt               r6d, hd
    sub                 tlq, hq
    tzcnt                wd, wm
    pxor                 m3, m3
    sub                 tlq, hq
    pavgw                m4, m3
    movd                 m5, r6d
    movu                 m0, [tlq]
    movsxd               r6, [r5+r6*4]
    add                  r6, r5
    add                  r5, ipred_dc_128_16bpc_ssse3_table-ipred_dc_left_16bpc_ssse3_table
    movsxd               wq, [r5+wq*4]
    add                  wq, r5
    jmp                  r6
.h64:
    movu                 m2, [tlq+112]
    movu                 m1, [tlq+ 96]
    paddw                m0, m2
    movu                 m2, [tlq+ 80]
    paddw                m1, m2
    movu                 m2, [tlq+ 64]
    paddw                m0, m2
    paddw                m0, m1
.h32:
    movu                 m1, [tlq+ 48]
    movu                 m2, [tlq+ 32]
    paddw                m1, m2
    paddw                m0, m1
.h16:
    movu                 m1, [tlq+ 16]
    paddw                m0, m1
.h8:
    movhlps              m1, m0
    paddw                m0, m1
.h4:
    punpcklwd            m0, m3
    paddd                m4, m0
    punpckhqdq           m0, m0
    paddd                m0, m4
    pshuflw              m4, m0, q1032
    paddd                m0, m4
    psrld                m0, m5
    lea            stride3q, [strideq*3]
    pshuflw              m0, m0, q0000
    punpcklqdq           m0, m0
    jmp                  wq

cglobal ipred_dc_16bpc, 4, 7, 6, dst, stride, tl, w, h, stride3
    movifnidn            hd, hm
    tzcnt               r6d, hd
    lea                 r5d, [wq+hq]
    movd                 m4, r5d
    tzcnt               r5d, r5d
    movd                 m5, r5d
    LEA                  r5, ipred_dc_16bpc_ssse3_table
    tzcnt                wd, wd
    movsxd               r6, [r5+r6*4]
    movsxd               wq, [r5+wq*4+5*4]
    pxor                 m3, m3
    psrlw                m4, 1
    add                  r6, r5
    add                  wq, r5
    lea            stride3q, [strideq*3]
    jmp                  r6
.h4:
    movq                 m0, [tlq-8]
    jmp                  wq
.w4:
    movq                 m1, [tlq+2]
    paddw                m1, m0
    punpckhwd            m0, m3
    punpcklwd            m1, m3
    paddd                m0, m1
    paddd                m4, m0
    punpckhqdq           m0, m0
    paddd                m0, m4
    pshuflw              m1, m0, q1032
    paddd                m0, m1
    cmp                  hd, 4
    jg .w4_mul
    psrlw                m0, 3
    jmp .w4_end
.w4_mul:
    mov                 r2d, 0xAAAB
    mov                 r3d, 0x6667
    cmp                  hd, 16
    cmove               r2d, r3d
    psrld                m0, 2
    movd                 m1, r2d
    pmulhuw              m0, m1
    psrlw                m0, 1
.w4_end:
    pshuflw              m0, m0, q0000
.s4:
    movq   [dstq+strideq*0], m0
    movq   [dstq+strideq*1], m0
    movq   [dstq+strideq*2], m0
    movq   [dstq+stride3q ], m0
    lea                dstq, [dstq+strideq*4]
    sub                  hd, 4
    jg .s4
    RET
.h8:
    mova                 m0, [tlq-16]
    jmp                  wq
.w8:
    movu                 m1, [tlq+2]
    paddw                m0, m1
    punpcklwd            m1, m0, m3
    punpckhwd            m0, m3
    paddd                m0, m1
    paddd                m4, m0
    punpckhqdq           m0, m0
    paddd                m0, m4
    pshuflw              m1, m0, q1032
    paddd                m0, m1
    psrld                m0, m5
    cmp                  hd, 8
    je .w8_end
    mov                 r2d, 0xAAAB
    mov                 r3d, 0x6667
    cmp                  hd, 32
    cmove               r2d, r3d
    movd                 m1, r2d
    pmulhuw              m0, m1
    psrlw                m0, 1
.w8_end:
    pshuflw              m0, m0, q0000
    punpcklqdq           m0, m0
.s8:
    mova   [dstq+strideq*0], m0
    mova   [dstq+strideq*1], m0
    mova   [dstq+strideq*2], m0
    mova   [dstq+stride3q ], m0
    lea                dstq, [dstq+strideq*4]
    sub                  hd, 4
    jg .s8
    RET
.h16:
    mova                 m0, [tlq-32]
    paddw                m0, [tlq-16]
    jmp                  wq
.w16:
    movu                 m1, [tlq+ 2]
    movu                 m2, [tlq+18]
    paddw                m1, m2
    paddw                m0, m1
    punpckhwd            m1, m0, m3
    punpcklwd            m0, m3
    paddd                m0, m1
    paddd                m4, m0
    punpckhqdq           m0, m0
    paddd                m0, m4
    pshuflw              m1, m0, q1032
    paddd                m0, m1
    psrld                m0, m5
    cmp                  hd, 16
    je .w16_end
    mov                 r2d, 0xAAAB
    mov                 r3d, 0x6667
    test                 hd, 8|32
    cmovz               r2d, r3d
    movd                 m1, r2d
    pmulhuw              m0, m1
    psrlw                m0, 1
.w16_end:
    pshuflw              m0, m0, q0000
    punpcklqdq           m0, m0
.s16c:
    mova                 m1, m0
.s16:
    mova [dstq+strideq*0+16*0], m0
    mova [dstq+strideq*0+16*1], m1
    mova [dstq+strideq*1+16*0], m0
    mova [dstq+strideq*1+16*1], m1
    mova [dstq+strideq*2+16*0], m0
    mova [dstq+strideq*2+16*1], m1
    mova [dstq+stride3q +16*0], m0
    mova [dstq+stride3q +16*1], m1
    lea                dstq, [dstq+strideq*4]
    sub                  hd, 4
    jg .s16
    RET
.h32:
    mova                 m0, [tlq-64]
    paddw                m0, [tlq-48]
    paddw                m0, [tlq-32]
    paddw                m0, [tlq-16]
    jmp                  wq
.w32:
    movu                 m1, [tlq+ 2]
    movu                 m2, [tlq+18]
    paddw                m1, m2
    movu                 m2, [tlq+34]
    paddw                m0, m2
    movu                 m2, [tlq+50]
    paddw                m1, m2
    paddw                m0, m1
    punpcklwd            m1, m0, m3
    punpckhwd            m0, m3
    paddd                m0, m1
    paddd                m4, m0
    punpckhqdq           m0, m0
    paddd                m0, m4
    pshuflw              m1, m0, q1032
    paddd                m0, m1
    psrld                m0, m5
    cmp                  hd, 32
    je .w32_end
    mov                 r2d, 0xAAAB
    mov                 r3d, 0x6667
    cmp                  hd, 8
    cmove               r2d, r3d
    movd                 m1, r2d
    pmulhuw              m0, m1
    psrlw                m0, 1
.w32_end:
    pshuflw              m0, m0, q0000
    punpcklqdq           m0, m0
.s32c:
    mova                 m1, m0
    mova                 m2, m0
    mova                 m3, m0
.s32:
    mova [dstq+strideq*0+16*0], m0
    mova [dstq+strideq*0+16*1], m1
    mova [dstq+strideq*0+16*2], m2
    mova [dstq+strideq*0+16*3], m3
    mova [dstq+strideq*1+16*0], m0
    mova [dstq+strideq*1+16*1], m1
    mova [dstq+strideq*1+16*2], m2
    mova [dstq+strideq*1+16*3], m3
    lea                dstq, [dstq+strideq*2]
    sub                  hd, 2
    jg .s32
    RET
.h64:
    mova                 m0, [tlq-128]
    mova                 m1, [tlq-112]
    paddw                m0, [tlq- 96]
    paddw                m1, [tlq- 80]
    paddw                m0, [tlq- 64]
    paddw                m1, [tlq- 48]
    paddw                m0, [tlq- 32]
    paddw                m1, [tlq- 16]
    paddw                m0, m1
    jmp                  wq
.w64:
    movu                 m1, [tlq+  2]
    movu                 m2, [tlq+ 18]
    paddw                m1, m2
    movu                 m2, [tlq+ 34]
    paddw                m0, m2
    movu                 m2, [tlq+ 50]
    paddw                m1, m2
    movu                 m2, [tlq+ 66]
    paddw                m0, m2
    movu                 m2, [tlq+ 82]
    paddw                m1, m2
    movu                 m2, [tlq+ 98]
    paddw                m0, m2
    movu                 m2, [tlq+114]
    paddw                m1, m2
    paddw                m0, m1
    punpcklwd            m1, m0, m3
    punpckhwd            m0, m3
    paddd                m0, m1
    paddd                m4, m0
    punpckhqdq           m0, m0
    paddd                m0, m4
    pshuflw              m1, m0, q1032
    paddd                m0, m1
    psrld                m0, m5
    cmp                  hd, 64
    je .w64_end
    mov                 r2d, 0xAAAB
    mov                 r3d, 0x6667
    cmp                  hd, 16
    cmove               r2d, r3d
    movd                 m1, r2d
    pmulhuw              m0, m1
    psrlw                m0, 1
.w64_end:
    pshuflw              m0, m0, q0000
    punpcklqdq           m0, m0
.s64:
    mova        [dstq+16*0], m0
    mova        [dstq+16*1], m0
    mova        [dstq+16*2], m0
    mova        [dstq+16*3], m0
    mova        [dstq+16*4], m0
    mova        [dstq+16*5], m0
    mova        [dstq+16*6], m0
    mova        [dstq+16*7], m0
    add                dstq, strideq
    dec                  hd
    jg .s64
    RET

cglobal ipred_dc_128_16bpc, 2, 7, 6, dst, stride, tl, w, h, stride3
    mov                 r6d, r8m
    LEA                  r5, ipred_dc_128_16bpc_ssse3_table
    tzcnt                wd, wm
    shr                 r6d, 11
    movifnidn            hd, hm
    movsxd               wq, [r5+wq*4]
    movddup              m0, [r5-ipred_dc_128_16bpc_ssse3_table+pw_512+r6*8]
    add                  wq, r5
    lea            stride3q, [strideq*3]
    jmp                  wq

cglobal ipred_v_16bpc, 4, 7, 6, dst, stride, tl, w, h, stride3
    LEA                  r5, ipred_dc_splat_16bpc_ssse3_table
    movifnidn            hd, hm
    movu                 m0, [tlq+  2]
    movu                 m1, [tlq+ 18]
    movu                 m2, [tlq+ 34]
    movu                 m3, [tlq+ 50]
    cmp                  wd, 64
    je .w64
    tzcnt                wd, wd
    movsxd               wq, [r5+wq*4]
    add                  wq, r5
    lea            stride3q, [strideq*3]
    jmp                  wq
.w64:
    WIN64_SPILL_XMM 8
    movu                 m4, [tlq+ 66]
    movu                 m5, [tlq+ 82]
    movu                 m6, [tlq+ 98]
    movu                 m7, [tlq+114]
.w64_loop:
    mova        [dstq+16*0], m0
    mova        [dstq+16*1], m1
    mova        [dstq+16*2], m2
    mova        [dstq+16*3], m3
    mova        [dstq+16*4], m4
    mova        [dstq+16*5], m5
    mova        [dstq+16*6], m6
    mova        [dstq+16*7], m7
    add                dstq, strideq
    dec                  hd
    jg .w64_loop
    RET

cglobal ipred_h_16bpc, 3, 6, 4, dst, stride, tl, w, h, stride3
%define base r5-ipred_h_16bpc_ssse3_table
    tzcnt                wd, wm
    LEA                  r5, ipred_h_16bpc_ssse3_table
    movifnidn            hd, hm
    movsxd               wq, [r5+wq*4]
    movddup              m2, [base+pb_0_1]
    movddup              m3, [base+pb_2_3]
    add                  wq, r5
    lea            stride3q, [strideq*3]
    jmp                  wq
.w4:
    sub                 tlq, 8
    movq                 m3, [tlq]
    pshuflw              m0, m3, q3333
    pshuflw              m1, m3, q2222
    pshuflw              m2, m3, q1111
    pshuflw              m3, m3, q0000
    movq   [dstq+strideq*0], m0
    movq   [dstq+strideq*1], m1
    movq   [dstq+strideq*2], m2
    movq   [dstq+stride3q ], m3
    lea                dstq, [dstq+strideq*4]
    sub                  hd, 4
    jg .w4
    RET
.w8:
    sub                 tlq, 8
    movq                 m3, [tlq]
    punpcklwd            m3, m3
    pshufd               m0, m3, q3333
    pshufd               m1, m3, q2222
    pshufd               m2, m3, q1111
    pshufd               m3, m3, q0000
    mova   [dstq+strideq*0], m0
    mova   [dstq+strideq*1], m1
    mova   [dstq+strideq*2], m2
    mova   [dstq+stride3q ], m3
    lea                dstq, [dstq+strideq*4]
    sub                  hd, 4
    jg .w8
    RET
.w16:
    sub                 tlq, 4
    movd                 m1, [tlq]
    pshufb               m0, m1, m3
    pshufb               m1, m2
    mova [dstq+strideq*0+16*0], m0
    mova [dstq+strideq*0+16*1], m0
    mova [dstq+strideq*1+16*0], m1
    mova [dstq+strideq*1+16*1], m1
    lea                dstq, [dstq+strideq*2]
    sub                  hd, 2
    jg .w16
    RET
.w32:
    sub                 tlq, 4
    movd                 m1, [tlq]
    pshufb               m0, m1, m3
    pshufb               m1, m2
    mova [dstq+strideq*0+16*0], m0
    mova [dstq+strideq*0+16*1], m0
    mova [dstq+strideq*0+16*2], m0
    mova [dstq+strideq*0+16*3], m0
    mova [dstq+strideq*1+16*0], m1
    mova [dstq+strideq*1+16*1], m1
    mova [dstq+strideq*1+16*2], m1
    mova [dstq+strideq*1+16*3], m1
    lea                dstq, [dstq+strideq*2]
    sub                  hd, 2
    jg .w32
    RET
.w64:
    sub                 tlq, 2
    movd                 m0, [tlq]
    pshufb               m0, m2
    mova        [dstq+16*0], m0
    mova        [dstq+16*1], m0
    mova        [dstq+16*2], m0
    mova        [dstq+16*3], m0
    mova        [dstq+16*4], m0
    mova        [dstq+16*5], m0
    mova        [dstq+16*6], m0
    mova        [dstq+16*7], m0
    add                dstq, strideq
    dec                  hd
    jg .w64
    RET

cglobal ipred_paeth_16bpc, 4, 6, 8, dst, stride, tl, w, h, left
%define base r5-ipred_paeth_16bpc_ssse3_table
    movifnidn            hd, hm
    pshuflw              m4, [tlq], q0000
    mov               leftq, tlq
    add                  hd, hd
    punpcklqdq           m4, m4      ; topleft
    sub               leftq, hq
    and                  wd, ~7
    jnz .w8
    movddup              m5, [tlq+2] ; top
    psubw                m6, m5, m4
    pabsw                m7, m6
.w4_loop:
    movd                 m1, [leftq+hq-4]
    punpcklwd            m1, m1
    punpckldq            m1, m1      ; left
%macro PAETH 0
    paddw                m0, m6, m1
    psubw                m2, m4, m0  ; tldiff
    psubw                m0, m5      ; tdiff
    pabsw                m2, m2
    pabsw                m0, m0
    pminsw               m2, m0
    pcmpeqw              m0, m2
    pand                 m3, m5, m0
    pandn                m0, m4
    por                  m0, m3
    pcmpgtw              m3, m7, m2
    pand                 m0, m3
    pandn                m3, m1
    por                  m0, m3
%endmacro
    PAETH
    movhps [dstq+strideq*0], m0
    movq   [dstq+strideq*1], m0
    lea                dstq, [dstq+strideq*2]
    sub                  hd, 2*2
    jg .w4_loop
    RET
.w8:
%if ARCH_X86_32
    PUSH                 r6
    %define             r7d  hm
    %assign regs_used     7
%elif WIN64
    movaps              r4m, m8
    PUSH                 r7
    %assign regs_used     8
%endif
%if ARCH_X86_64
    movddup              m8, [pb_0_1]
%endif
    lea                 tlq, [tlq+wq*2+2]
    neg                  wq
    mov                 r7d, hd
.w8_loop0:
    movu                 m5, [tlq+wq*2]
    mov                  r6, dstq
    add                dstq, 16
    psubw                m6, m5, m4
    pabsw                m7, m6
.w8_loop:
    movd                 m1, [leftq+hq-2]
%if ARCH_X86_64
    pshufb               m1, m8
%else
    pshuflw              m1, m1, q0000
    punpcklqdq           m1, m1
%endif
    PAETH
    mova               [r6], m0
    add                  r6, strideq
    sub                  hd, 1*2
    jg .w8_loop
    mov                  hd, r7d
    add                  wq, 8
    jl .w8_loop0
%if WIN64
    movaps               m8, r4m
%endif
    RET

%if ARCH_X86_64
DECLARE_REG_TMP 7
%else
DECLARE_REG_TMP 4
%endif

cglobal ipred_smooth_v_16bpc, 4, 6, 6, dst, stride, tl, w, h, weights
    LEA            weightsq, smooth_weights_1d_16bpc
    mov                  hd, hm
    lea            weightsq, [weightsq+hq*4]
    neg                  hq
    movd                 m5, [tlq+hq*2] ; bottom
    pshuflw              m5, m5, q0000
    punpcklqdq           m5, m5
    cmp                  wd, 4
    jne .w8
    movddup              m4, [tlq+2]    ; top
    lea                  r3, [strideq*3]
    psubw                m4, m5         ; top - bottom
.w4_loop:
    movq                 m1, [weightsq+hq*2]
    punpcklwd            m1, m1
    pshufd               m0, m1, q1100
    punpckhdq            m1, m1
    pmulhrsw             m0, m4
    pmulhrsw             m1, m4
    paddw                m0, m5
    paddw                m1, m5
    movq   [dstq+strideq*0], m0
    movhps [dstq+strideq*1], m0
    movq   [dstq+strideq*2], m1
    movhps [dstq+r3       ], m1
    lea                dstq, [dstq+strideq*4]
    add                  hq, 4
    jl .w4_loop
    RET
.w8:
%if ARCH_X86_32
    PUSH                 r6
    %assign regs_used     7
    mov                  hm, hq
    %define              hq  hm
%elif WIN64
    PUSH                 r7
    %assign regs_used     8
%endif
.w8_loop0:
    mov                  t0, hq
    movu                 m4, [tlq+2]
    add                 tlq, 16
    mov                  r6, dstq
    add                dstq, 16
    psubw                m4, m5
.w8_loop:
    movq                 m3, [weightsq+t0*2]
    punpcklwd            m3, m3
    pshufd               m0, m3, q0000
    pshufd               m1, m3, q1111
    pshufd               m2, m3, q2222
    pshufd               m3, m3, q3333
    REPX   {pmulhrsw x, m4}, m0, m1, m2, m3
    REPX   {paddw    x, m5}, m0, m1, m2, m3
    mova     [r6+strideq*0], m0
    mova     [r6+strideq*1], m1
    lea                  r6, [r6+strideq*2]
    mova     [r6+strideq*0], m2
    mova     [r6+strideq*1], m3
    lea                  r6, [r6+strideq*2]
    add                  t0, 4
    jl .w8_loop
    sub                  wd, 8
    jg .w8_loop0
    RET

cglobal ipred_smooth_h_16bpc, 3, 6, 6, dst, stride, tl, w, h, weights
    LEA            weightsq, smooth_weights_1d_16bpc
    mov                  wd, wm
    movifnidn            hd, hm
    movd                 m5, [tlq+wq*2] ; right
    sub                 tlq, 8
    add                  hd, hd
    pshuflw              m5, m5, q0000
    sub                 tlq, hq
    punpcklqdq           m5, m5
    cmp                  wd, 4
    jne .w8
    movddup              m4, [weightsq+4*2]
    lea                  r3, [strideq*3]
.w4_loop:
    movq                 m1, [tlq+hq]   ; left
    punpcklwd            m1, m1
    psubw                m1, m5         ; left - right
    pshufd               m0, m1, q3322
    punpckldq            m1, m1
    pmulhrsw             m0, m4
    pmulhrsw             m1, m4
    paddw                m0, m5
    paddw                m1, m5
    movhps [dstq+strideq*0], m0
    movq   [dstq+strideq*1], m0
    movhps [dstq+strideq*2], m1
    movq   [dstq+r3       ], m1
    lea                dstq, [dstq+strideq*4]
    sub                  hd, 4*2
    jg .w4_loop
    RET
.w8:
    lea            weightsq, [weightsq+wq*4]
    neg                  wq
%if ARCH_X86_32
    PUSH                 r6
    %assign regs_used     7
    %define              hd  hm
%elif WIN64
    PUSH                 r7
    %assign regs_used     8
%endif
.w8_loop0:
    mov                 t0d, hd
    mova                 m4, [weightsq+wq*2]
    mov                  r6, dstq
    add                dstq, 16
.w8_loop:
    movq                 m3, [tlq+t0*(1+ARCH_X86_32)]
    punpcklwd            m3, m3
    psubw                m3, m5
    pshufd               m0, m3, q3333
    pshufd               m1, m3, q2222
    pshufd               m2, m3, q1111
    pshufd               m3, m3, q0000
    REPX   {pmulhrsw x, m4}, m0, m1, m2, m3
    REPX   {paddw    x, m5}, m0, m1, m2, m3
    mova     [r6+strideq*0], m0
    mova     [r6+strideq*1], m1
    lea                  r6, [r6+strideq*2]
    mova     [r6+strideq*0], m2
    mova     [r6+strideq*1], m3
    lea                  r6, [r6+strideq*2]
    sub                 t0d, 4*(1+ARCH_X86_64)
    jg .w8_loop
    add                  wq, 8
    jl .w8_loop0
    RET

%if ARCH_X86_64
DECLARE_REG_TMP 10
%else
DECLARE_REG_TMP 3
%endif

cglobal ipred_smooth_16bpc, 3, 7, 8, dst, stride, tl, w, h, \
                                     h_weights, v_weights, top
    LEA          h_weightsq, smooth_weights_2d_16bpc
    mov                  wd, wm
    mov                  hd, hm
    movd                 m7, [tlq+wq*2] ; right
    lea          v_weightsq, [h_weightsq+hq*8]
    neg                  hq
    movd                 m6, [tlq+hq*2] ; bottom
    pshuflw              m7, m7, q0000
    pshuflw              m6, m6, q0000
    cmp                  wd, 4
    jne .w8
    movq                 m4, [tlq+2]    ; top
    mova                 m5, [h_weightsq+4*4]
    punpcklwd            m4, m6         ; top, bottom
    pxor                 m6, m6
.w4_loop:
    movq                 m1, [v_weightsq+hq*4]
    sub                 tlq, 4
    movd                 m3, [tlq]      ; left
    pshufd               m0, m1, q0000
    pshufd               m1, m1, q1111
    pmaddwd              m0, m4
    punpcklwd            m3, m7         ; left, right
    pmaddwd              m1, m4
    pshufd               m2, m3, q1111
    pshufd               m3, m3, q0000
    pmaddwd              m2, m5
    pmaddwd              m3, m5
    paddd                m0, m2
    paddd                m1, m3
    psrld                m0, 8
    psrld                m1, 8
    packssdw             m0, m1
    pavgw                m0, m6
    movq   [dstq+strideq*0], m0
    movhps [dstq+strideq*1], m0
    lea                dstq, [dstq+strideq*2]
    add                  hq, 2
    jl .w4_loop
    RET
.w8:
%if ARCH_X86_32
    lea          h_weightsq, [h_weightsq+wq*4]
    mov                  t0, tlq
    mov                 r1m, tlq
    mov                 r2m, hq
    %define              m8  [h_weightsq+16*0]
    %define              m9  [h_weightsq+16*1]
%else
%if WIN64
    movaps              r4m, m8
    movaps              r6m, m9
    PUSH                 r7
    PUSH                 r8
%endif
    PUSH                 r9
    PUSH                r10
    %assign       regs_used  11
    lea          h_weightsq, [h_weightsq+wq*8]
    lea                topq, [tlq+wq*2]
    neg                  wq
    mov                  r8, tlq
    mov                  r9, hq
%endif
    punpcklqdq           m6, m6
.w8_loop0:
%if ARCH_X86_32
    movu                 m5, [t0+2]
    add                  t0, 16
    mov                 r0m, t0
%else
    movu                 m5, [topq+wq*2+2]
    mova                 m8, [h_weightsq+wq*4+16*0]
    mova                 m9, [h_weightsq+wq*4+16*1]
%endif
    mov                  t0, dstq
    add                dstq, 16
    punpcklwd            m4, m5, m6
    punpckhwd            m5, m6
.w8_loop:
    movd                 m1, [v_weightsq+hq*4]
    sub                 tlq, 2
    movd                 m3, [tlq]      ; left
    pshufd               m1, m1, q0000
    pmaddwd              m0, m4, m1
    pshuflw              m3, m3, q0000
    pmaddwd              m1, m5
    punpcklwd            m3, m7         ; left, right
    pmaddwd              m2, m8, m3
    pmaddwd              m3, m9
    paddd                m0, m2
    paddd                m1, m3
    psrld                m0, 8
    psrld                m1, 8
    packssdw             m0, m1
    pxor                 m1, m1
    pavgw                m0, m1
    mova               [t0], m0
    add                  t0, strideq
    inc                  hq
    jl .w8_loop
%if ARCH_X86_32
    mov                  t0, r0m
    mov                 tlq, r1m
    add          h_weightsq, 16*2
    mov                  hq, r2m
    sub            dword wm, 8
    jg .w8_loop0
%else
    mov                 tlq, r8
    mov                  hq, r9
    add                  wq, 8
    jl .w8_loop0
%endif
%if WIN64
    movaps               m8, r4m
    movaps               m9, r6m
%endif
    RET

%if ARCH_X86_64
cglobal ipred_filter_16bpc, 4, 7, 16, dst, stride, tl, w, h, filter
%else
cglobal ipred_filter_16bpc, 4, 7, 8, -16*8, dst, stride, tl, w, h, filter
%define  m8 [esp+16*0]
%define  m9 [esp+16*1]
%define m10 [esp+16*2]
%define m11 [esp+16*3]
%define m12 [esp+16*4]
%define m13 [esp+16*5]
%define m14 [esp+16*6]
%define m15 [esp+16*7]
%endif
%define base r6-$$
    movifnidn            hd, hm
    movd                 m6, r8m     ; bitdepth_max
%ifidn filterd, filterm
    movzx           filterd, filterb
%else
    movzx           filterd, byte filterm
%endif
    LEA                  r6, $$
    shl             filterd, 6
    movu                 m0, [tlq-6] ; __ l1 l0 tl t0 t1 t2 t3
    mova                 m1, [base+filter_intra_taps+filterq+16*0]
    mova                 m2, [base+filter_intra_taps+filterq+16*1]
    mova                 m3, [base+filter_intra_taps+filterq+16*2]
    mova                 m4, [base+filter_intra_taps+filterq+16*3]
    pxor                 m5, m5
%if ARCH_X86_64
    punpcklbw            m8, m5, m1  ; place 8-bit coefficients in the upper
    punpckhbw            m9, m5, m1  ; half of each 16-bit word to avoid
    punpcklbw           m10, m5, m2  ; having to perform sign-extension.
    punpckhbw           m11, m5, m2
    punpcklbw           m12, m5, m3
    punpckhbw           m13, m5, m3
    punpcklbw           m14, m5, m4
    punpckhbw           m15, m5, m4
%else
    punpcklbw            m7, m5, m1
    mova                 m8, m7
    punpckhbw            m7, m5, m1
    mova                 m9, m7
    punpcklbw            m7, m5, m2
    mova                m10, m7
    punpckhbw            m7, m5, m2
    mova                m11, m7
    punpcklbw            m7, m5, m3
    mova                m12, m7
    punpckhbw            m7, m5, m3
    mova                m13, m7
    punpcklbw            m7, m5, m4
    mova                m14, m7
    punpckhbw            m7, m5, m4
    mova                m15, m7
%endif
    mova                 m7, [base+filter_shuf]
    add                  hd, hd
    mov                  r5, dstq
    pshuflw              m6, m6, q0000
    mov                  r6, tlq
    punpcklqdq           m6, m6
    sub                 tlq, hq
.left_loop:
    pshufb               m0, m7      ; tl t0 t1 t2 t3 l0 l1 __
    pshufd               m1, m0, q0000
    pmaddwd              m2, m8, m1
    pmaddwd              m1, m9
    pshufd               m4, m0, q1111
    pmaddwd              m3, m10, m4
    pmaddwd              m4, m11
    paddd                m2, m3
    paddd                m1, m4
    pshufd               m4, m0, q2222
    pmaddwd              m3, m12, m4
    pmaddwd              m4, m13
    paddd                m2, m3
    paddd                m1, m4
    pshufd               m3, m0, q3333
    pmaddwd              m0, m14, m3
    pmaddwd              m3, m15
    paddd                m0, m2
    paddd                m1, m3
    psrad                m0, 11     ; x >> 3
    psrad                m1, 11
    packssdw             m0, m1
    pmaxsw               m0, m5
    pavgw                m0, m5     ; (x + 8) >> 4
    pminsw               m0, m6
    movq   [dstq+strideq*0], m0
    movhps [dstq+strideq*1], m0
    movlps               m0, [tlq+hq-10]
    lea                dstq, [dstq+strideq*2]
    sub                  hd, 2*2
    jg .left_loop
    sub                  wd, 4
    jz .end
    sub                 tld, r6d     ; -h*2
    sub                  r6, r5      ; tl-dst
.right_loop0:
    add                  r5, 8
    mov                  hd, tld
    movu                 m0, [r5+r6] ; tl t0 t1 t2 t3 __ __ __
    mov                dstq, r5
.right_loop:
    pshufd               m2, m0, q0000
    pmaddwd              m1, m8, m2
    pmaddwd              m2, m9
    pshufd               m4, m0, q1111
    pmaddwd              m3, m10, m4
    pmaddwd              m4, m11
    pinsrw               m0, [dstq+strideq*0-2], 5
    paddd                m1, m3
    paddd                m2, m4
    pshufd               m0, m0, q2222
    movddup              m4, [dstq+strideq*1-8]
    pmaddwd              m3, m12, m0
    pmaddwd              m0, m13
    paddd                m1, m3
    paddd                m0, m2
    pshuflw              m2, m4, q3333
    punpcklwd            m2, m5
    pmaddwd              m3, m14, m2
    pmaddwd              m2, m15
    paddd                m1, m3
    paddd                m0, m2
    psrad                m1, 11
    psrad                m0, 11
    packssdw             m0, m1
    pmaxsw               m0, m5
    pavgw                m0, m5
    pminsw               m0, m6
    movhps [dstq+strideq*0], m0
    movq   [dstq+strideq*1], m0
    palignr              m0, m4, 14
    lea                dstq, [dstq+strideq*2]
    add                  hd, 2*2
    jl .right_loop
    sub                  wd, 4
    jg .right_loop0
.end:
    RET

%if UNIX64
DECLARE_REG_TMP 7
%else
DECLARE_REG_TMP 5
%endif

cglobal ipred_cfl_top_16bpc, 4, 7, 8, dst, stride, tl, w, h, ac
    LEA                  t0, ipred_cfl_left_16bpc_ssse3_table
    movd                 m4, wd
    tzcnt                wd, wd
    movifnidn            hd, hm
    add                 tlq, 2
    movsxd               r6, [t0+wq*4]
    movd                 m5, wd
    jmp mangle(private_prefix %+ _ipred_cfl_left_16bpc_ssse3.start)

cglobal ipred_cfl_left_16bpc, 3, 7, 8, dst, stride, tl, w, h, ac, alpha
    movifnidn            hd, hm
    LEA                  t0, ipred_cfl_left_16bpc_ssse3_table
    tzcnt                wd, wm
    lea                 r6d, [hq*2]
    movd                 m4, hd
    sub                 tlq, r6
    tzcnt               r6d, hd
    movd                 m5, r6d
    movsxd               r6, [t0+r6*4]
.start:
    movd                 m7, r7m
    movu                 m0, [tlq]
    add                  r6, t0
    add                  t0, ipred_cfl_splat_16bpc_ssse3_table-ipred_cfl_left_16bpc_ssse3_table
    movsxd               wq, [t0+wq*4]
    pxor                 m6, m6
    pshuflw              m7, m7, q0000
    pcmpeqw              m3, m3
    add                  wq, t0
    movifnidn           acq, acmp
    pavgw                m4, m6
    punpcklqdq           m7, m7
    jmp                  r6
.h32:
    movu                 m1, [tlq+48]
    movu                 m2, [tlq+32]
    paddw                m0, m1
    paddw                m0, m2
.h16:
    movu                 m1, [tlq+16]
    paddw                m0, m1
.h8:
    pshufd               m1, m0, q1032
    paddw                m0, m1
.h4:
    pmaddwd              m0, m3
    psubd                m4, m0
    pshuflw              m0, m4, q1032
    paddd                m0, m4
    psrld                m0, m5
    pshuflw              m0, m0, q0000
    punpcklqdq           m0, m0
    jmp                  wq

%macro IPRED_CFL 2 ; dst, src
    pabsw               m%1, m%2
    pmulhrsw            m%1, m2
    psignw              m%2, m1
    psignw              m%1, m%2
    paddw               m%1, m0
    pmaxsw              m%1, m6
    pminsw              m%1, m7
%endmacro

cglobal ipred_cfl_16bpc, 4, 7, 8, dst, stride, tl, w, h, ac, alpha
    movifnidn            hd, hm
    tzcnt               r6d, hd
    lea                 t0d, [wq+hq]
    movd                 m4, t0d
    tzcnt               t0d, t0d
    movd                 m5, t0d
    LEA                  t0, ipred_cfl_16bpc_ssse3_table
    tzcnt                wd, wd
    movd                 m7, r7m
    movsxd               r6, [t0+r6*4]
    movsxd               wq, [t0+wq*4+4*4]
    psrlw                m4, 1
    pxor                 m6, m6
    pshuflw              m7, m7, q0000
    add                  r6, t0
    add                  wq, t0
    movifnidn           acq, acmp
    pcmpeqw              m3, m3
    punpcklqdq           m7, m7
    jmp                  r6
.h4:
    movq                 m0, [tlq-8]
    jmp                  wq
.w4:
    movq                 m1, [tlq+2]
    paddw                m0, m1
    pmaddwd              m0, m3
    psubd                m4, m0
    pshufd               m0, m4, q1032
    paddd                m0, m4
    pshuflw              m4, m0, q1032
    paddd                m0, m4
    cmp                  hd, 4
    jg .w4_mul
    psrld                m0, 3
    jmp .w4_end
.w4_mul:
    mov                 r6d, 0xAAAB
    mov                 r2d, 0x6667
    cmp                  hd, 16
    cmove               r6d, r2d
    movd                 m1, r6d
    psrld                m0, 2
    pmulhuw              m0, m1
    psrlw                m0, 1
.w4_end:
    pshuflw              m0, m0, q0000
    punpcklqdq           m0, m0
.s4:
    movd                 m1, alpham
    lea                  r6, [strideq*3]
    pshuflw              m1, m1, q0000
    punpcklqdq           m1, m1
    pabsw                m2, m1
    psllw                m2, 9
.s4_loop:
    mova                 m4, [acq+16*0]
    mova                 m5, [acq+16*1]
    add                 acq, 16*2
    IPRED_CFL             3, 4
    IPRED_CFL             4, 5
    movq   [dstq+strideq*0], m3
    movhps [dstq+strideq*1], m3
    movq   [dstq+strideq*2], m4
    movhps [dstq+r6       ], m4
    lea                dstq, [dstq+strideq*4]
    sub                  hd, 4
    jg .s4_loop
    RET
.h8:
    mova                 m0, [tlq-16]
    jmp                  wq
.w8:
    movu                 m1, [tlq+2]
    paddw                m0, m1
    pmaddwd              m0, m3
    psubd                m4, m0
    pshufd               m0, m4, q1032
    paddd                m0, m4
    pshuflw              m4, m0, q1032
    paddd                m0, m4
    psrld                m0, m5
    cmp                  hd, 8
    je .w8_end
    mov                 r6d, 0xAAAB
    mov                 r2d, 0x6667
    cmp                  hd, 32
    cmove               r6d, r2d
    movd                 m1, r6d
    pmulhuw              m0, m1
    psrlw                m0, 1
.w8_end:
    pshuflw              m0, m0, q0000
    punpcklqdq           m0, m0
.s8:
    movd                 m1, alpham
    pshuflw              m1, m1, q0000
    punpcklqdq           m1, m1
    pabsw                m2, m1
    psllw                m2, 9
.s8_loop:
    mova                 m4, [acq+16*0]
    mova                 m5, [acq+16*1]
    add                 acq, 16*2
    IPRED_CFL             3, 4
    IPRED_CFL             4, 5
    mova   [dstq+strideq*0], m3
    mova   [dstq+strideq*1], m4
    lea                dstq, [dstq+strideq*2]
    sub                  hd, 2
    jg .s8_loop
    RET
.h16:
    mova                 m0, [tlq-32]
    paddw                m0, [tlq-16]
    jmp                  wq
.w16:
    movu                 m1, [tlq+ 2]
    movu                 m2, [tlq+18]
    paddw                m1, m2
    paddw                m0, m1
    pmaddwd              m0, m3
    psubd                m4, m0
    pshufd               m0, m4, q1032
    paddd                m0, m4
    pshuflw              m4, m0, q1032
    paddd                m0, m4
    psrld                m0, m5
    cmp                  hd, 16
    je .w16_end
    mov                 r6d, 0xAAAB
    mov                 r2d, 0x6667
    test                 hd, 8|32
    cmovz               r6d, r2d
    movd                 m1, r6d
    pmulhuw              m0, m1
    psrlw                m0, 1
.w16_end:
    pshuflw              m0, m0, q0000
    punpcklqdq           m0, m0
.s16:
    movd                 m1, alpham
    pshuflw              m1, m1, q0000
    punpcklqdq           m1, m1
    pabsw                m2, m1
    psllw                m2, 9
.s16_loop:
    mova                 m4, [acq+16*0]
    mova                 m5, [acq+16*1]
    add                 acq, 16*2
    IPRED_CFL             3, 4
    IPRED_CFL             4, 5
    mova        [dstq+16*0], m3
    mova        [dstq+16*1], m4
    add                dstq, strideq
    dec                  hd
    jg .s16_loop
    RET
.h32:
    mova                 m0, [tlq-64]
    paddw                m0, [tlq-48]
    paddw                m0, [tlq-32]
    paddw                m0, [tlq-16]
    jmp                  wq
.w32:
    movu                 m1, [tlq+ 2]
    movu                 m2, [tlq+18]
    paddw                m1, m2
    movu                 m2, [tlq+34]
    paddw                m1, m2
    movu                 m2, [tlq+50]
    paddw                m1, m2
    paddw                m0, m1
    pmaddwd              m0, m3
    psubd                m4, m0
    pshufd               m0, m4, q1032
    paddd                m0, m4
    pshuflw              m4, m0, q1032
    paddd                m0, m4
    psrld                m0, m5
    cmp                  hd, 32
    je .w32_end
    mov                 r6d, 0xAAAB
    mov                 r2d, 0x6667
    cmp                  hd, 8
    cmove               r6d, r2d
    movd                 m1, r6d
    pmulhuw              m0, m1
    psrlw                m0, 1
.w32_end:
    pshuflw              m0, m0, q0000
    punpcklqdq           m0, m0
.s32:
    movd                 m1, alpham
    pshuflw              m1, m1, q0000
    punpcklqdq           m1, m1
    pabsw                m2, m1
    psllw                m2, 9
.s32_loop:
    mova                 m4, [acq+16*0]
    mova                 m5, [acq+16*1]
    IPRED_CFL             3, 4
    IPRED_CFL             4, 5
    mova        [dstq+16*0], m3
    mova        [dstq+16*1], m4
    mova                 m4, [acq+16*2]
    mova                 m5, [acq+16*3]
    add                 acq, 16*4
    IPRED_CFL             3, 4
    IPRED_CFL             4, 5
    mova        [dstq+16*2], m3
    mova        [dstq+16*3], m4
    add                dstq, strideq
    dec                  hd
    jg .s32_loop
    RET

cglobal ipred_cfl_128_16bpc, 3, 7, 8, dst, stride, tl, w, h, ac
    tzcnt                wd, wm
    LEA                  t0, ipred_cfl_splat_16bpc_ssse3_table
    mov                 r6d, r7m
    movifnidn            hd, hm
    shr                 r6d, 11
    movd                 m7, r7m
    movsxd               wq, [t0+wq*4]
    movddup              m0, [t0-ipred_cfl_splat_16bpc_ssse3_table+pw_512+r6*8]
    pshuflw              m7, m7, q0000
    pxor                 m6, m6
    add                  wq, t0
    movifnidn           acq, acmp
    punpcklqdq           m7, m7
    jmp                  wq

cglobal ipred_cfl_ac_420_16bpc, 3, 7, 6, ac, ypx, stride, wpad, hpad, w, h
    movifnidn         hpadd, hpadm
%if ARCH_X86_32 && PIC
    pcmpeqw              m5, m5
    pabsw                m5, m5
    paddw                m5, m5
%else
    movddup              m5, [pw_2]
%endif
    mov                  hd, hm
    shl               hpadd, 2
    pxor                 m4, m4
    sub                  hd, hpadd
    cmp            dword wm, 8
    mov                  r5, acq
    jg .w16
    je .w8
    lea                  r3, [strideq*3]
.w4_loop:
    pmaddwd              m0, m5, [ypxq+strideq*0]
    pmaddwd              m1, m5, [ypxq+strideq*1]
    pmaddwd              m2, m5, [ypxq+strideq*2]
    pmaddwd              m3, m5, [ypxq+r3       ]
    lea                ypxq, [ypxq+strideq*4]
    paddd                m0, m1
    paddd                m2, m3
    paddd                m4, m0
    packssdw             m0, m2
    paddd                m4, m2
    mova              [acq], m0
    add                 acq, 16
    sub                  hd, 2
    jg .w4_loop
    test              hpadd, hpadd
    jz .dc
    punpckhqdq           m0, m0
    pslld                m2, 2
.w4_hpad:
    mova         [acq+16*0], m0
    paddd                m4, m2
    mova         [acq+16*1], m0
    add                 acq, 16*2
    sub               hpadd, 4
    jg .w4_hpad
    jmp .dc
.w8:
%if ARCH_X86_32
    cmp         dword wpadm, 0
%else
    test              wpadd, wpadd
%endif
    jnz .w8_wpad1
.w8_loop:
    pmaddwd              m0, m5, [ypxq+strideq*0+16*0]
    pmaddwd              m2, m5, [ypxq+strideq*1+16*0]
    pmaddwd              m1, m5, [ypxq+strideq*0+16*1]
    pmaddwd              m3, m5, [ypxq+strideq*1+16*1]
    lea                ypxq, [ypxq+strideq*2]
    paddd                m0, m2
    paddd                m1, m3
    paddd                m2, m0, m1
    packssdw             m0, m1
    paddd                m4, m2
    mova              [acq], m0
    add                 acq, 16
    dec                  hd
    jg .w8_loop
.w8_hpad:
    test              hpadd, hpadd
    jz .dc
    pslld                m2, 2
    mova                 m1, m0
    jmp .hpad
.w8_wpad1:
    pmaddwd              m0, m5, [ypxq+strideq*0]
    pmaddwd              m1, m5, [ypxq+strideq*1]
    lea                ypxq, [ypxq+strideq*2]
    paddd                m0, m1
    pshufd               m1, m0, q3333
    paddd                m2, m0, m1
    packssdw             m0, m1
    paddd                m4, m2
    mova              [acq], m0
    add                 acq, 16
    dec                  hd
    jg .w8_wpad1
    jmp .w8_hpad
.w16_wpad3:
    pshufd               m3, m0, q3333
    mova                 m1, m3
    mova                 m2, m3
    jmp .w16_wpad_end
.w16_wpad2:
    pshufd               m1, m3, q3333
    mova                 m2, m1
    jmp .w16_wpad_end
.w16_wpad1:
    pshufd               m2, m1, q3333
    jmp .w16_wpad_end
.w16:
    movifnidn         wpadd, wpadm
    WIN64_SPILL_XMM       7
.w16_loop:
    pmaddwd              m0, m5, [ypxq+strideq*0+16*0]
    pmaddwd              m6, m5, [ypxq+strideq*1+16*0]
    paddd                m0, m6
    cmp               wpadd, 2
    jg .w16_wpad3
    pmaddwd              m3, m5, [ypxq+strideq*0+16*1]
    pmaddwd              m6, m5, [ypxq+strideq*1+16*1]
    paddd                m3, m6
    je .w16_wpad2
    pmaddwd              m1, m5, [ypxq+strideq*0+16*2]
    pmaddwd              m6, m5, [ypxq+strideq*1+16*2]
    paddd                m1, m6
    jp .w16_wpad1
    pmaddwd              m2, m5, [ypxq+strideq*0+16*3]
    pmaddwd              m6, m5, [ypxq+strideq*1+16*3]
    paddd                m2, m6
.w16_wpad_end:
    lea                ypxq, [ypxq+strideq*2]
    paddd                m6, m0, m3
    packssdw             m0, m3
    paddd                m6, m1
    mova         [acq+16*0], m0
    packssdw             m1, m2
    paddd                m2, m6
    mova         [acq+16*1], m1
    add                 acq, 16*2
    paddd                m4, m2
    dec                  hd
    jg .w16_loop
    WIN64_RESTORE_XMM
    add               hpadd, hpadd
    jz .dc
    paddd                m2, m2
.hpad:
    mova         [acq+16*0], m0
    mova         [acq+16*1], m1
    paddd                m4, m2
    mova         [acq+16*2], m0
    mova         [acq+16*3], m1
    add                 acq, 16*4
    sub               hpadd, 4
    jg .hpad
.dc:
    sub                  r5, acq ; -w*h*2
    pshufd               m2, m4, q1032
    tzcnt               r1d, r5d
    paddd                m2, m4
    sub                 r1d, 2
    pshufd               m4, m2, q2301
    movd                 m0, r1d
    paddd                m2, m4
    psrld                m2, m0
    pxor                 m0, m0
    pavgw                m2, m0
    packssdw             m2, m2
.dc_loop:
    mova                 m0, [acq+r5+16*0]
    mova                 m1, [acq+r5+16*1]
    psubw                m0, m2
    psubw                m1, m2
    mova      [acq+r5+16*0], m0
    mova      [acq+r5+16*1], m1
    add                  r5, 16*2
    jl .dc_loop
    RET

cglobal ipred_cfl_ac_422_16bpc, 3, 7, 6, ac, ypx, stride, wpad, hpad, w, h
    movifnidn         hpadd, hpadm
%if ARCH_X86_32 && PIC
    pcmpeqw              m5, m5
    pabsw                m5, m5
    psllw                m5, 2
%else
    movddup              m5, [pw_4]
%endif
    mov                  hd, hm
    shl               hpadd, 2
    pxor                 m4, m4
    sub                  hd, hpadd
    cmp            dword wm, 8
    mov                  r5, acq
    jg .w16
    je .w8
    lea                  r3, [strideq*3]
.w4_loop:
    pmaddwd              m0, m5, [ypxq+strideq*0]
    pmaddwd              m3, m5, [ypxq+strideq*1]
    pmaddwd              m1, m5, [ypxq+strideq*2]
    pmaddwd              m2, m5, [ypxq+r3       ]
    lea                ypxq, [ypxq+strideq*4]
    paddd                m4, m0
    packssdw             m0, m3
    paddd                m3, m1
    packssdw             m1, m2
    paddd                m4, m2
    paddd                m4, m3
    mova         [acq+16*0], m0
    mova         [acq+16*1], m1
    add                 acq, 16*2
    sub                  hd, 4
    jg .w4_loop
    test              hpadd, hpadd
    jz mangle(private_prefix %+ _ipred_cfl_ac_420_16bpc_ssse3).dc
    punpckhqdq           m1, m1
    pslld                m2, 3
    mova         [acq+16*0], m1
    mova         [acq+16*1], m1
    paddd                m4, m2
    mova         [acq+16*2], m1
    mova         [acq+16*3], m1
    add                 acq, 16*4
    jmp mangle(private_prefix %+ _ipred_cfl_ac_420_16bpc_ssse3).dc
.w8:
%if ARCH_X86_32
    cmp         dword wpadm, 0
%else
    test              wpadd, wpadd
%endif
    jnz .w8_wpad1
.w8_loop:
    pmaddwd              m0, m5, [ypxq+strideq*0+16*0]
    pmaddwd              m2, m5, [ypxq+strideq*0+16*1]
    pmaddwd              m1, m5, [ypxq+strideq*1+16*0]
    pmaddwd              m3, m5, [ypxq+strideq*1+16*1]
    lea                ypxq, [ypxq+strideq*2]
    paddd                m4, m0
    packssdw             m0, m2
    paddd                m4, m2
    mova         [acq+16*0], m0
    paddd                m2, m1, m3
    packssdw             m1, m3
    paddd                m4, m2
    mova         [acq+16*1], m1
    add                 acq, 16*2
    sub                  hd, 2
    jg .w8_loop
.w8_hpad:
    test              hpadd, hpadd
    jz mangle(private_prefix %+ _ipred_cfl_ac_420_16bpc_ssse3).dc
    pslld                m2, 2
    mova                 m0, m1
    jmp mangle(private_prefix %+ _ipred_cfl_ac_420_16bpc_ssse3).hpad
.w8_wpad1:
    pmaddwd              m0, m5, [ypxq+strideq*0]
    pmaddwd              m1, m5, [ypxq+strideq*1]
    lea                ypxq, [ypxq+strideq*2]
    pshufd               m2, m0, q3333
    pshufd               m3, m1, q3333
    paddd                m4, m0
    packssdw             m0, m2
    paddd                m4, m2
    paddd                m2, m1, m3
    packssdw             m1, m3
    paddd                m4, m2
    mova         [acq+16*0], m0
    mova         [acq+16*1], m1
    add                 acq, 16*2
    sub                  hd, 2
    jg .w8_wpad1
    jmp .w8_hpad
.w16_wpad3:
    pshufd               m3, m0, q3333
    mova                 m1, m3
    mova                 m2, m3
    jmp .w16_wpad_end
.w16_wpad2:
    pshufd               m1, m3, q3333
    mova                 m2, m1
    jmp .w16_wpad_end
.w16_wpad1:
    pshufd               m2, m1, q3333
    jmp .w16_wpad_end
.w16:
    movifnidn         wpadd, wpadm
    WIN64_SPILL_XMM       7
.w16_loop:
    pmaddwd              m0, m5, [ypxq+16*0]
    cmp               wpadd, 2
    jg .w16_wpad3
    pmaddwd              m3, m5, [ypxq+16*1]
    je .w16_wpad2
    pmaddwd              m1, m5, [ypxq+16*2]
    jp .w16_wpad1
    pmaddwd              m2, m5, [ypxq+16*3]
.w16_wpad_end:
    add                ypxq, strideq
    paddd                m6, m0, m3
    packssdw             m0, m3
    mova         [acq+16*0], m0
    paddd                m6, m1
    packssdw             m1, m2
    paddd                m2, m6
    mova         [acq+16*1], m1
    add                 acq, 16*2
    paddd                m4, m2
    dec                  hd
    jg .w16_loop
    WIN64_RESTORE_XMM
    add               hpadd, hpadd
    jz mangle(private_prefix %+ _ipred_cfl_ac_420_16bpc_ssse3).dc
    paddd                m2, m2
    jmp mangle(private_prefix %+ _ipred_cfl_ac_420_16bpc_ssse3).hpad

cglobal ipred_cfl_ac_444_16bpc, 3, 7, 6, ac, ypx, stride, wpad, hpad, w, h
%define base r6-ipred_cfl_ac_444_16bpc_ssse3_table
    LEA                  r6, ipred_cfl_ac_444_16bpc_ssse3_table
    tzcnt                wd, wm
    movifnidn         hpadd, hpadm
    pxor                 m4, m4
    movsxd               wq, [r6+wq*4]
    movddup              m5, [base+pw_1]
    add                  wq, r6
    mov                  hd, hm
    shl               hpadd, 2
    sub                  hd, hpadd
    jmp                  wq
.w4:
    lea                  r3, [strideq*3]
    mov                  r5, acq
.w4_loop:
    movq                 m0, [ypxq+strideq*0]
    movhps               m0, [ypxq+strideq*1]
    movq                 m1, [ypxq+strideq*2]
    movhps               m1, [ypxq+r3       ]
    lea                ypxq, [ypxq+strideq*4]
    psllw                m0, 3
    psllw                m1, 3
    mova         [acq+16*0], m0
    pmaddwd              m0, m5
    mova         [acq+16*1], m1
    pmaddwd              m2, m5, m1
    add                 acq, 16*2
    paddd                m4, m0
    paddd                m4, m2
    sub                  hd, 4
    jg .w4_loop
    test              hpadd, hpadd
    jz mangle(private_prefix %+ _ipred_cfl_ac_420_16bpc_ssse3).dc
    punpckhqdq           m1, m1
    mova         [acq+16*0], m1
    pslld                m2, 2
    mova         [acq+16*1], m1
    punpckhqdq           m2, m2
    mova         [acq+16*2], m1
    paddd                m4, m2
    mova         [acq+16*3], m1
    add                 acq, 16*4
    jmp mangle(private_prefix %+ _ipred_cfl_ac_420_16bpc_ssse3).dc
.w8:
    mov                  r5, acq
.w8_loop:
    mova                 m0, [ypxq+strideq*0]
    mova                 m1, [ypxq+strideq*1]
    lea                ypxq, [ypxq+strideq*2]
    psllw                m0, 3
    psllw                m1, 3
    mova         [acq+16*0], m0
    pmaddwd              m0, m5
    mova         [acq+16*1], m1
    pmaddwd              m2, m5, m1
    add                 acq, 16*2
    paddd                m4, m0
    paddd                m4, m2
    sub                  hd, 2
    jg .w8_loop
.w8_hpad:
    test              hpadd, hpadd
    jz mangle(private_prefix %+ _ipred_cfl_ac_420_16bpc_ssse3).dc
    pslld                m2, 2
    mova                 m0, m1
    jmp mangle(private_prefix %+ _ipred_cfl_ac_420_16bpc_ssse3).hpad
.w16_wpad2:
    pshufhw              m3, m2, q3333
    pshufhw              m1, m0, q3333
    punpckhqdq           m3, m3
    punpckhqdq           m1, m1
    jmp .w16_wpad_end
.w16:
    movifnidn         wpadd, wpadm
    mov                  r5, acq
.w16_loop:
    mova                 m2, [ypxq+strideq*0+16*0]
    mova                 m0, [ypxq+strideq*1+16*0]
    psllw                m2, 3
    psllw                m0, 3
    test              wpadd, wpadd
    jnz .w16_wpad2
    mova                 m3, [ypxq+strideq*0+16*1]
    mova                 m1, [ypxq+strideq*1+16*1]
    psllw                m3, 3
    psllw                m1, 3
.w16_wpad_end:
    lea                ypxq, [ypxq+strideq*2]
    mova         [acq+16*0], m2
    pmaddwd              m2, m5
    mova         [acq+16*1], m3
    pmaddwd              m3, m5
    paddd                m4, m2
    pmaddwd              m2, m5, m0
    mova         [acq+16*2], m0
    paddd                m4, m3
    pmaddwd              m3, m5, m1
    mova         [acq+16*3], m1
    add                 acq, 16*4
    paddd                m2, m3
    paddd                m4, m2
    sub                  hd, 2
    jg .w16_loop
    add               hpadd, hpadd
    jz mangle(private_prefix %+ _ipred_cfl_ac_420_16bpc_ssse3).dc
    paddd                m2, m2
    jmp mangle(private_prefix %+ _ipred_cfl_ac_420_16bpc_ssse3).hpad
.w32_wpad6:
    pshufhw              m1, m0, q3333
    punpckhqdq           m1, m1
    mova                 m2, m1
    mova                 m3, m1
    jmp .w32_wpad_end
.w32_wpad4:
    pshufhw              m2, m1, q3333
    punpckhqdq           m2, m2
    mova                 m3, m2
    jmp .w32_wpad_end
.w32_wpad2:
    pshufhw              m3, m2, q3333
    punpckhqdq           m3, m3
    jmp .w32_wpad_end
.w32:
    movifnidn         wpadd, wpadm
    mov                  r5, acq
    WIN64_SPILL_XMM       8
.w32_loop:
    mova                 m0, [ypxq+16*0]
    psllw                m0, 3
    cmp               wpadd, 4
    jg .w32_wpad6
    mova                 m1, [ypxq+16*1]
    psllw                m1, 3
    je .w32_wpad4
    mova                 m2, [ypxq+16*2]
    psllw                m2, 3
    jnp .w32_wpad2
    mova                 m3, [ypxq+16*3]
    psllw                m3, 3
.w32_wpad_end:
    add                ypxq, strideq
    pmaddwd              m6, m5, m0
    mova         [acq+16*0], m0
    pmaddwd              m7, m5, m1
    mova         [acq+16*1], m1
    paddd                m6, m7
    pmaddwd              m7, m5, m2
    mova         [acq+16*2], m2
    paddd                m6, m7
    pmaddwd              m7, m5, m3
    mova         [acq+16*3], m3
    add                 acq, 16*4
    paddd                m6, m7
    paddd                m4, m6
    dec                  hd
    jg .w32_loop
%if WIN64
    mova                 m5, m6
    WIN64_RESTORE_XMM
    SWAP                  5, 6
%endif
    test              hpadd, hpadd
    jz mangle(private_prefix %+ _ipred_cfl_ac_420_16bpc_ssse3).dc
.w32_hpad_loop:
    mova         [acq+16*0], m0
    mova         [acq+16*1], m1
    paddd                m4, m6
    mova         [acq+16*2], m2
    mova         [acq+16*3], m3
    add                 acq, 16*4
    dec               hpadd
    jg .w32_hpad_loop
    jmp mangle(private_prefix %+ _ipred_cfl_ac_420_16bpc_ssse3).dc

cglobal pal_pred_16bpc, 4, 5, 5, dst, stride, pal, idx, w, h
%define base r2-pal_pred_16bpc_ssse3_table
%if ARCH_X86_32
    %define              hd  r2d
%endif
    mova                 m3, [palq]
    LEA                  r2, pal_pred_16bpc_ssse3_table
    tzcnt                wd, wm
    pshufb               m3, [base+pal_pred_shuf]
    movsxd               wq, [r2+wq*4]
    pshufd               m4, m3, q1032
    add                  wq, r2
    movifnidn            hd, hm
    jmp                  wq
.w4:
    mova                 m0, [idxq]
    add                idxq, 16
    pshufb               m1, m3, m0
    pshufb               m2, m4, m0
    punpcklbw            m0, m1, m2
    punpckhbw            m1, m2
    movq   [dstq+strideq*0], m0
    movhps [dstq+strideq*1], m0
    lea                dstq, [dstq+strideq*2]
    movq   [dstq+strideq*0], m1
    movhps [dstq+strideq*1], m1
    lea                dstq, [dstq+strideq*2]
    sub                  hd, 4
    jg .w4
    RET
.w8:
    mova                 m0, [idxq]
    add                idxq, 16
    pshufb               m1, m3, m0
    pshufb               m2, m4, m0
    punpcklbw            m0, m1, m2
    punpckhbw            m1, m2
    mova   [dstq+strideq*0], m0
    mova   [dstq+strideq*1], m1
    lea                dstq, [dstq+strideq*2]
    sub                  hd, 2
    jg .w8
    RET
.w16:
    mova                 m0, [idxq]
    add                idxq, 16
    pshufb               m1, m3, m0
    pshufb               m2, m4, m0
    punpcklbw            m0, m1, m2
    punpckhbw            m1, m2
    mova        [dstq+16*0], m0
    mova        [dstq+16*1], m1
    add                dstq, strideq
    dec                  hd
    jg .w16
    RET
.w32:
    mova                 m0, [idxq+16*0]
    pshufb               m1, m3, m0
    pshufb               m2, m4, m0
    punpcklbw            m0, m1, m2
    punpckhbw            m1, m2
    mova                 m2, [idxq+16*1]
    add                idxq, 16*2
    mova        [dstq+16*0], m0
    pshufb               m0, m3, m2
    mova        [dstq+16*1], m1
    pshufb               m1, m4, m2
    punpcklbw            m2, m0, m1
    punpckhbw            m0, m1
    mova        [dstq+16*2], m2
    mova        [dstq+16*3], m0
    add                dstq, strideq
    dec                  hd
    jg .w32
    RET
.w64:
    mova                 m0, [idxq+16*0]
    pshufb               m1, m3, m0
    pshufb               m2, m4, m0
    punpcklbw            m0, m1, m2
    punpckhbw            m1, m2
    mova                 m2, [idxq+16*1]
    mova        [dstq+16*0], m0
    pshufb               m0, m3, m2
    mova        [dstq+16*1], m1
    pshufb               m1, m4, m2
    punpcklbw            m2, m0, m1
    punpckhbw            m0, m1
    mova                 m1, [idxq+16*2]
    mova        [dstq+16*2], m2
    pshufb               m2, m3, m1
    mova        [dstq+16*3], m0
    pshufb               m0, m4, m1
    punpcklbw            m1, m2, m0
    punpckhbw            m2, m0
    mova                 m0, [idxq+16*3]
    add                idxq, 16*4
    mova        [dstq+16*4], m1
    pshufb               m1, m3, m0
    mova        [dstq+16*5], m2
    pshufb               m2, m4, m0
    punpcklbw            m0, m1, m2
    punpckhbw            m1, m2
    mova        [dstq+16*6], m0
    mova        [dstq+16*7], m1
    add                dstq, strideq
    dec                  hd
    jg .w64
    RET