diff options
Diffstat (limited to 'third_party/dav1d/src')
| -rw-r--r-- | third_party/dav1d/src/arm/64/mc.S | 4 | ||||
| -rw-r--r-- | third_party/dav1d/src/arm/64/mc_dotprod.S | 1413 | ||||
| -rw-r--r-- | third_party/dav1d/src/arm/64/msac.S | 21 | ||||
| -rw-r--r-- | third_party/dav1d/src/arm/itx.h | 63 | ||||
| -rw-r--r-- | third_party/dav1d/src/arm/mc.h | 85 | ||||
| -rw-r--r-- | third_party/dav1d/src/cdf.c | 1378 | ||||
| -rw-r--r-- | third_party/dav1d/src/cdf.h | 48 | ||||
| -rw-r--r-- | third_party/dav1d/src/decode.c | 95 | ||||
| -rw-r--r-- | third_party/dav1d/src/internal.h | 9 | ||||
| -rw-r--r-- | third_party/dav1d/src/itx.h | 63 | ||||
| -rw-r--r-- | third_party/dav1d/src/lf_mask.c | 6 | ||||
| -rw-r--r-- | third_party/dav1d/src/meson.build | 1 | ||||
| -rw-r--r-- | third_party/dav1d/src/refmvs.c | 4 | ||||
| -rw-r--r-- | third_party/dav1d/src/riscv/itx.h | 63 | ||||
| -rw-r--r-- | third_party/dav1d/src/x86/ipred_avx2.asm | 3 | ||||
| -rw-r--r-- | third_party/dav1d/src/x86/itx.h | 64 | ||||
| -rw-r--r-- | third_party/dav1d/src/x86/mc16_avx2.asm | 1378 | ||||
| -rw-r--r-- | third_party/dav1d/src/x86/mc_avx2.asm | 1471 | ||||
| -rw-r--r-- | third_party/dav1d/src/x86/mc_avx512.asm | 2953 |
19 files changed, 6598 insertions, 2524 deletions
diff --git a/third_party/dav1d/src/arm/64/mc.S b/third_party/dav1d/src/arm/64/mc.S index 3df0393c3a..5b493be82d 100644 --- a/third_party/dav1d/src/arm/64/mc.S +++ b/third_party/dav1d/src/arm/64/mc.S @@ -837,7 +837,7 @@ endfunc // This has got the same signature as the put_8tap functions, // and assumes that x8 is set to (clz(w)-24). -function put_neon +function put_neon, export=1 adr x9, L(put_tbl) ldrh w8, [x9, x8, lsl #1] sub x9, x9, w8, uxtw @@ -939,7 +939,7 @@ endfunc // This has got the same signature as the prep_8tap functions, // and assumes that x8 is set to (clz(w)-24), and x7 to w*2. -function prep_neon +function prep_neon, export=1 adr x9, L(prep_tbl) ldrh w8, [x9, x8, lsl #1] sub x9, x9, w8, uxtw diff --git a/third_party/dav1d/src/arm/64/mc_dotprod.S b/third_party/dav1d/src/arm/64/mc_dotprod.S new file mode 100644 index 0000000000..fcf04ee4d0 --- /dev/null +++ b/third_party/dav1d/src/arm/64/mc_dotprod.S @@ -0,0 +1,1413 @@ +/* + * Copyright © 2024, VideoLAN and dav1d authors + * Copyright © 2024, Janne Grunau + * Copyright © 2024, Martin Storsjo + * Copyright © 2024, Arm Limited + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "src/arm/asm.S" +#include "util.S" + + +#if HAVE_DOTPROD +ENABLE_DOTPROD + +// No spaces in these expressions, due to gas-preprocessor. It is translated by +// -1 to save the negative offset at getting the address of `mc_subpel_filters`. +#define REGULAR1 (((0*15-1)<<7)|(3*15-1)) +#define SMOOTH1 (((1*15-1)<<7)|(4*15-1)) +#define SHARP1 (((2*15-1)<<7)|(3*15-1)) + +#define FUNC_ALIGN 2 +#define JUMP_ALIGN 2 +#define LOOP_ALIGN 2 + + +// Lookup table used to help conversion of shifted 32-bit values to 8-bit. + .align 4 +L(hv_tbl_neon_dotprod): + .byte 1, 2, 5, 6, 9, 10, 13, 14, 17, 18, 21, 22, 25, 26, 29, 30 + +// Shuffle indices to permute horizontal samples in preparation for input to +// SDOT instructions. The 8-tap horizontal convolution uses sample indices in the +// interval of [-3, 4] relative to the current sample position. We load samples +// from index value -4 to keep loads word aligned, so the shuffle bytes are +// translated by 1 to handle this. + .align 4 +L(h_tbl_neon_dotprod): + .byte 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6, 4, 5, 6, 7 + .byte 5, 6, 7, 8, 6, 7, 8, 9, 7, 8, 9, 10, 8, 9, 10, 11 + .byte 9, 10, 11, 12, 10, 11, 12, 13, 11, 12, 13, 14, 12, 13, 14, 15 + .byte 13, 14, 15, 16, 14, 15, 16, 17, 15, 16, 17, 18, 16, 17, 18, 19 + +// Vertical convolutions are also using SDOT instructions, where a 128-bit +// register contains a transposed 4x4 matrix of values. Subsequent iterations of +// the vertical convolution can reuse the 3x4 sub-matrix from the previous loop +// iteration. These shuffle indices shift and merge this 4x4 matrix with the +// values of a new line. + .align 4 +L(v_tbl_neon_dotprod): + .byte 1, 2, 3, 16, 5, 6, 7, 20, 9, 10, 11, 24, 13, 14, 15, 28 + .byte 1, 2, 3, 16, 5, 6, 7, 17, 9, 10, 11, 18, 13, 14, 15, 19 + .byte 1, 2, 3, 20, 5, 6, 7, 21, 9, 10, 11, 22, 13, 14, 15, 23 + .byte 1, 2, 3, 24, 5, 6, 7, 25, 9, 10, 11, 26, 13, 14, 15, 27 + .byte 1, 2, 3, 28, 5, 6, 7, 29, 9, 10, 11, 30, 13, 14, 15, 31 + + +.macro make_8tap_fn op, type, type_h, type_v, isa, jump=1 +function \op\()_8tap_\type\()_8bpc_\isa, export=1, align=FUNC_ALIGN + mov x9, \type_h + mov x10, \type_v + .if \jump + b \op\()_8tap_\isa + .endif +endfunc +.endm + +.macro filter_8tap_fn type, isa, dst, d_strd, src, s_strd, w, h, mx, my, xmx, xmy, ldst, lsrc, wd_strd +make_8tap_fn \type, sharp, SHARP1, SHARP1, \isa +make_8tap_fn \type, sharp_smooth, SHARP1, SMOOTH1, \isa +make_8tap_fn \type, sharp_regular, SHARP1, REGULAR1, \isa +make_8tap_fn \type, smooth_sharp, SMOOTH1, SHARP1, \isa +make_8tap_fn \type, smooth, SMOOTH1, SMOOTH1, \isa +make_8tap_fn \type, smooth_regular, SMOOTH1, REGULAR1, \isa +make_8tap_fn \type, regular_sharp, REGULAR1, SHARP1, \isa +make_8tap_fn \type, regular_smooth, REGULAR1, SMOOTH1, \isa +make_8tap_fn \type, regular, REGULAR1, REGULAR1, \isa, jump=0 + +function \type\()_8tap_\isa, align=FUNC_ALIGN + clz w8, \w + mov w11, #0x4081 // (1 << 14) | (1 << 7) | (1 << 0) + sub w8, w8, #24 // for jump tables + movrel x12, X(mc_subpel_filters) + cbnz \mx, L(\type\()_8tap_h_hv_\isa) + cbnz \my, L(\type\()_8tap_v_\isa) +.ifc \type, prep + add \wd_strd, \w, \w // prep_neon needs w * 2 as stride +.endif + b X(\type\()_neon) + + .align JUMP_ALIGN +L(\type\()_8tap_v_\isa): + madd \my, \my, w11, w10 +.ifc \type, prep + mov w8, 0x2002 // FILTER_WEIGHT * 128 + rounding +.endif + sub \src, \src, \s_strd + ldr q6, L(v_tbl_neon_dotprod) +.ifc \type, prep + dup v4.4s, w8 +.endif + ubfx w11, \my, #7, #7 + and \my, \my, #0x7F + ldr q28, L(v_tbl_neon_dotprod) + 16 + cmp \h, #4 + csel \my, \my, w11, le + sub \src, \src, \s_strd, lsl #1 // src - src_stride * 3 + ldr q29, L(v_tbl_neon_dotprod) + 32 + add \xmy, x12, \xmy, lsl #3 // subpel V filter address + movi v5.16b, #128 + ldr d7, [\xmy] + cmp \w, #8 + b.eq 80f + b.lt 40f + + // .align JUMP_ALIGN // fallthrough +160: // V - 16xN+ + ldr q30, L(v_tbl_neon_dotprod) + 48 + ldr q31, L(v_tbl_neon_dotprod) + 64 +.ifc \type, prep + add \wd_strd, \w, \w +.endif + .align LOOP_ALIGN +161: + mov \lsrc, \src + mov \ldst, \dst + sub w8, \h, #1 + + ldr q16, [\lsrc] + ldr q17, [\lsrc, \s_strd] + add \lsrc, \lsrc, \s_strd, lsl #1 + ldr q18, [\lsrc] + ldr q19, [\lsrc, \s_strd] + add \lsrc, \lsrc, \s_strd, lsl #1 + + zip1 v0.16b, v16.16b, v17.16b + zip2 v1.16b, v16.16b, v17.16b + zip1 v2.16b, v18.16b, v19.16b + zip2 v3.16b, v18.16b, v19.16b + + ldr q20, [\lsrc] + ldr q21, [\lsrc, \s_strd] + add \lsrc, \lsrc, \s_strd, lsl #1 + ldr q22, [\lsrc] + ldr q23, [\lsrc, \s_strd] + add \lsrc, \lsrc, \s_strd, lsl #1 + + zip1 v18.16b, v20.16b, v21.16b + zip2 v21.16b, v20.16b, v21.16b + zip1 v24.16b, v22.16b, v23.16b + zip2 v27.16b, v22.16b, v23.16b + + zip1 v16.8h, v0.8h, v2.8h + zip2 v19.8h, v0.8h, v2.8h + zip1 v22.8h, v1.8h, v3.8h + zip2 v25.8h, v1.8h, v3.8h + + zip1 v17.8h, v18.8h, v24.8h + zip2 v20.8h, v18.8h, v24.8h + zip1 v23.8h, v21.8h, v27.8h + zip2 v26.8h, v21.8h, v27.8h + + sub v16.16b, v16.16b, v5.16b + sub v19.16b, v19.16b, v5.16b + sub v22.16b, v22.16b, v5.16b + sub v25.16b, v25.16b, v5.16b + + sub v17.16b, v17.16b, v5.16b + sub v20.16b, v20.16b, v5.16b + sub v23.16b, v23.16b, v5.16b + sub v26.16b, v26.16b, v5.16b + + .align LOOP_ALIGN +16: + ldr q27, [\lsrc] + add \lsrc, \lsrc, \s_strd +.ifc \type, prep + mov v0.16b, v4.16b + mov v1.16b, v4.16b + mov v2.16b, v4.16b + mov v3.16b, v4.16b +.else + movi v0.4s, #32, lsl 8 // 64 * 128, bias for SDOT + movi v1.4s, #32, lsl 8 + movi v2.4s, #32, lsl 8 + movi v3.4s, #32, lsl 8 +.endif + sub v18.16b, v27.16b, v5.16b + sub v21.16b, v27.16b, v5.16b + sub v24.16b, v27.16b, v5.16b + sub v27.16b, v27.16b, v5.16b + + sdot v0.4s, v16.16b, v7.4b[0] + sdot v1.4s, v19.16b, v7.4b[0] + sdot v2.4s, v22.16b, v7.4b[0] + sdot v3.4s, v25.16b, v7.4b[0] + + tbl v16.16b, {v16.16b, v17.16b}, v6.16b + tbl v19.16b, {v19.16b, v20.16b}, v6.16b + tbl v22.16b, {v22.16b, v23.16b}, v6.16b + tbl v25.16b, {v25.16b, v26.16b}, v6.16b + + sdot v0.4s, v17.16b, v7.4b[1] + sdot v1.4s, v20.16b, v7.4b[1] + sdot v2.4s, v23.16b, v7.4b[1] + sdot v3.4s, v26.16b, v7.4b[1] + + tbl v17.16b, {v17.16b, v18.16b}, v28.16b + tbl v20.16b, {v20.16b, v21.16b}, v29.16b + tbl v23.16b, {v23.16b, v24.16b}, v30.16b + tbl v26.16b, {v26.16b, v27.16b}, v31.16b + + subs w8, w8, #1 + uzp1 v0.8h, v0.8h, v1.8h + uzp1 v2.8h, v2.8h, v3.8h +.ifc \type, prep + sshr v0.8h, v0.8h, #2 + sshr v1.8h, v2.8h, #2 + st1 {v0.8h, v1.8h}, [\ldst], \d_strd +.else + sqrshrun v0.8b, v0.8h, #6 + sqrshrun2 v0.16b, v2.8h, #6 + st1 {v0.16b}, [\ldst], \d_strd +.endif + b.gt 16b + +.ifc \type, prep + mov v0.16b, v4.16b + mov v1.16b, v4.16b + mov v2.16b, v4.16b + mov v3.16b, v4.16b +.else + movi v0.4s, #32, lsl 8 // 64 * 128, bias for SDOT + movi v1.4s, #32, lsl 8 + movi v2.4s, #32, lsl 8 + movi v3.4s, #32, lsl 8 +.endif + sdot v0.4s, v16.16b, v7.4b[0] + sdot v1.4s, v19.16b, v7.4b[0] + sdot v2.4s, v22.16b, v7.4b[0] + sdot v3.4s, v25.16b, v7.4b[0] + + sdot v0.4s, v17.16b, v7.4b[1] + sdot v1.4s, v20.16b, v7.4b[1] + sdot v2.4s, v23.16b, v7.4b[1] + sdot v3.4s, v26.16b, v7.4b[1] + + subs \w, \w, #16 + uzp1 v0.8h, v0.8h, v1.8h + uzp1 v2.8h, v2.8h, v3.8h +.ifc \type, prep + sshr v0.8h, v0.8h, #2 + sshr v1.8h, v2.8h, #2 + stp q0, q1, [\ldst] + add \dst, \dst, #32 +.else + sqrshrun v0.8b, v0.8h, #6 + sqrshrun2 v0.16b, v2.8h, #6 + str q0, [\ldst] + add \dst, \dst, #16 +.endif + add \src, \src, #16 + b.gt 161b + ret + + .align JUMP_ALIGN +80: // V - 8xN + ldr d16, [\src] + ldr d17, [\src, \s_strd] + add \src, \src, \s_strd, lsl #1 + ldr d18, [\src] + ldr d19, [\src, \s_strd] + add \src, \src, \s_strd, lsl #1 + + ldr d20, [\src] + ldr d21, [\src, \s_strd] + add \src, \src, \s_strd, lsl #1 + ldr d22, [\src] + ldr d23, [\src, \s_strd] + add \src, \src, \s_strd, lsl #1 + subs \h, \h, #2 // for prep: sub is enough + + zip1 v0.16b, v16.16b, v17.16b + zip1 v2.16b, v18.16b, v19.16b + zip1 v18.16b, v20.16b, v21.16b + zip1 v24.16b, v22.16b, v23.16b + + zip1 v16.8h, v0.8h, v2.8h + zip2 v19.8h, v0.8h, v2.8h + zip1 v17.8h, v18.8h, v24.8h + zip2 v20.8h, v18.8h, v24.8h + + sub v16.16b, v16.16b, v5.16b + sub v19.16b, v19.16b, v5.16b + sub v17.16b, v17.16b, v5.16b + sub v20.16b, v20.16b, v5.16b +.ifc \type, put + b.eq 82f +.endif + + .align LOOP_ALIGN +8: + ldr d21, [\src] + ldr d27, [\src, \s_strd] + add \src, \src, \s_strd, lsl #1 +.ifc \type, prep + mov v0.16b, v4.16b + mov v1.16b, v4.16b + mov v2.16b, v4.16b + mov v3.16b, v4.16b +.else + movi v0.4s, #32, lsl 8 // 64 * 128, bias for SDOT + movi v1.4s, #32, lsl 8 + movi v2.4s, #32, lsl 8 + movi v3.4s, #32, lsl 8 +.endif + sub v18.16b, v21.16b, v5.16b + sub v21.16b, v21.16b, v5.16b + sub v24.16b, v27.16b, v5.16b + sub v27.16b, v27.16b, v5.16b + + tbl v22.16b, {v16.16b, v17.16b}, v6.16b + tbl v25.16b, {v19.16b, v20.16b}, v6.16b + tbl v23.16b, {v17.16b, v18.16b}, v28.16b + tbl v26.16b, {v20.16b, v21.16b}, v29.16b + + sdot v0.4s, v16.16b, v7.4b[0] + sdot v0.4s, v17.16b, v7.4b[1] + sdot v1.4s, v19.16b, v7.4b[0] + sdot v1.4s, v20.16b, v7.4b[1] + + tbl v16.16b, {v22.16b, v23.16b}, v6.16b + tbl v19.16b, {v25.16b, v26.16b}, v6.16b + tbl v17.16b, {v23.16b, v24.16b}, v28.16b + tbl v20.16b, {v26.16b, v27.16b}, v29.16b + + sdot v2.4s, v22.16b, v7.4b[0] + sdot v2.4s, v23.16b, v7.4b[1] + sdot v3.4s, v25.16b, v7.4b[0] + sdot v3.4s, v26.16b, v7.4b[1] + + subs \h, \h, #2 + uzp1 v0.8h, v0.8h, v1.8h + uzp1 v2.8h, v2.8h, v3.8h +.ifc \type, prep + sshr v0.8h, v0.8h, #2 + sshr v1.8h, v2.8h, #2 + stp q0, q1, [\dst], #32 +.else + sqrshrun v0.8b, v0.8h, #6 + sqrshrun v1.8b, v2.8h, #6 + str d0, [\dst] + str d1, [\dst, \d_strd] + add \dst, \dst, \d_strd, lsl #1 +.endif + b.gt 8b + +.ifc \type, put + .align JUMP_ALIGN +82: + ldr d21, [\src] + movi v0.4s, #32, lsl 8 // 64 * 128, bias for SDOT + movi v1.4s, #32, lsl 8 + movi v2.4s, #32, lsl 8 + movi v3.4s, #32, lsl 8 +.else + ldr d21, [\src] + mov v0.16b, v4.16b + mov v1.16b, v4.16b + mov v2.16b, v4.16b + mov v3.16b, v4.16b +.endif + sub v18.16b, v21.16b, v5.16b + sub v21.16b, v21.16b, v5.16b + + tbl v22.16b, {v16.16b, v17.16b}, v6.16b + tbl v25.16b, {v19.16b, v20.16b}, v6.16b + tbl v23.16b, {v17.16b, v18.16b}, v28.16b + tbl v26.16b, {v20.16b, v21.16b}, v29.16b + + sdot v0.4s, v16.16b, v7.4b[0] + sdot v0.4s, v17.16b, v7.4b[1] + sdot v1.4s, v19.16b, v7.4b[0] + sdot v1.4s, v20.16b, v7.4b[1] + + sdot v2.4s, v22.16b, v7.4b[0] + sdot v2.4s, v23.16b, v7.4b[1] + sdot v3.4s, v25.16b, v7.4b[0] + sdot v3.4s, v26.16b, v7.4b[1] + + uzp1 v0.8h, v0.8h, v1.8h + uzp1 v2.8h, v2.8h, v3.8h +.ifc \type, prep + sshr v0.8h, v0.8h, #2 + sshr v1.8h, v2.8h, #2 + stp q0, q1, [\dst] +.else + sqrshrun v0.8b, v0.8h, #6 + sqrshrun v1.8b, v2.8h, #6 + str d0, [\dst] + str d1, [\dst, \d_strd] +.endif + ret + + .align JUMP_ALIGN +40: // V - 4xN or 2xN (put only) +.ifc \type, put + cmp \w, #2 + b.eq 20f +.endif + ldr s16, [\src] + ldr s17, [\src, \s_strd] + add \src, \src, \s_strd, lsl #1 + ldr s18, [\src] + ldr s19, [\src, \s_strd] + add \src, \src, \s_strd, lsl #1 + + ldr s20, [\src] + ldr s21, [\src, \s_strd] + add \src, \src, \s_strd, lsl #1 + ldr s22, [\src] + ldr s23, [\src, \s_strd] + add \src, \src, \s_strd, lsl #1 + subs \h, \h, #2 // for prep: sub is enough + + zip1 v0.8b, v16.8b, v17.8b + zip1 v2.8b, v18.8b, v19.8b + zip1 v18.8b, v20.8b, v21.8b + zip1 v24.8b, v22.8b, v23.8b + + zip1 v16.8h, v0.8h, v2.8h + zip1 v17.8h, v18.8h, v24.8h + + sub v16.16b, v16.16b, v5.16b + sub v17.16b, v17.16b, v5.16b +.ifc \type, put + b.eq 42f +.endif + + .align LOOP_ALIGN +4: + ldr s18, [\src] + ldr s21, [\src, \s_strd] + add \src, \src, \s_strd, lsl #1 +.ifc \type, prep + mov v0.16b, v4.16b + mov v1.16b, v4.16b +.else + movi v0.4s, #32, lsl 8 // 64 * 128, bias for SDOT + movi v1.4s, #32, lsl 8 +.endif + sub v18.16b, v18.16b, v5.16b + sub v21.16b, v21.16b, v5.16b + + tbl v19.16b, {v16.16b, v17.16b}, v6.16b + tbl v20.16b, {v17.16b, v18.16b}, v28.16b + + sdot v0.4s, v16.16b, v7.4b[0] + sdot v0.4s, v17.16b, v7.4b[1] + + tbl v16.16b, {v19.16b, v20.16b}, v6.16b + tbl v17.16b, {v20.16b, v21.16b}, v28.16b + + sdot v1.4s, v19.16b, v7.4b[0] + sdot v1.4s, v20.16b, v7.4b[1] +.ifc \type, prep + subs \h, \h, #2 + shrn v0.4h, v0.4s, #2 + shrn2 v0.8h, v1.4s, #2 + str q0, [\dst], #16 +.else + uzp1 v0.8h, v0.8h, v1.8h + sqrshrun v0.8b, v0.8h, #6 + subs \h, \h, #2 + fmov x8, d0 + lsr x9, x8, #32 + str w8, [\dst] + str w9, [\dst, \d_strd] + add \dst, \dst, \d_strd, lsl #1 +.endif + b.gt 4b + +.ifc \type, put + .align JUMP_ALIGN +42: + ldr s18, [\src] + movi v0.4s, #32, lsl 8 // 64 * 128, bias for SDOT + movi v1.4s, #32, lsl 8 +.else + ldr s18, [\src] + mov v0.16b, v4.16b + mov v1.16b, v4.16b +.endif + sub v18.16b, v18.16b, v5.16b + + tbl v19.16b, {v16.16b, v17.16b}, v6.16b + tbl v20.16b, {v17.16b, v18.16b}, v28.16b + + sdot v0.4s, v16.16b, v7.4b[0] + sdot v0.4s, v17.16b, v7.4b[1] + + sdot v1.4s, v19.16b, v7.4b[0] + sdot v1.4s, v20.16b, v7.4b[1] +.ifc \type, prep + shrn v0.4h, v0.4s, #2 + shrn2 v0.8h, v1.4s, #2 + str q0, [\dst] + ret +.else + uzp1 v0.8h, v0.8h, v1.8h + sqrshrun v0.8b, v0.8h, #6 + fmov x8, d0 + lsr x9, x8, #32 + str w8, [\dst] + str w9, [\dst, \d_strd] + ret + + .align JUMP_ALIGN +20: // V - 2xN + ldr h16, [\src] + ldr h17, [\src, \s_strd] + add \src, \src, \s_strd, lsl #1 + ldr h18, [\src] + ldr h19, [\src, \s_strd] + add \src, \src, \s_strd, lsl #1 + + ldr h20, [\src] + ldr h21, [\src, \s_strd] + add \src, \src, \s_strd, lsl #1 + ldr h22, [\src] + ldr h23, [\src, \s_strd] + add \src, \src, \s_strd, lsl #1 + subs \h, \h, #2 + + zip1 v0.8b, v16.8b, v17.8b + zip1 v2.8b, v18.8b, v19.8b + zip1 v18.8b, v20.8b, v21.8b + zip1 v24.8b, v22.8b, v23.8b + + zip1 v16.4h, v0.4h, v2.4h + zip1 v17.4h, v18.4h, v24.4h + + sub v16.8b, v16.8b, v5.8b + sub v17.8b, v17.8b, v5.8b + + b.eq 22f + + .align LOOP_ALIGN +2: + ldr h18, [\src] + ldr h21, [\src, \s_strd] + add \src, \src, \s_strd, lsl #1 + + movi v0.4s, #32, lsl 8 // 64 * 128, bias for SDOT + movi v1.4s, #32, lsl 8 + + sub v18.8b, v18.8b, v5.8b + sub v21.8b, v21.8b, v5.8b + + tbl v19.16b, {v16.16b, v17.16b}, v6.16b + tbl v20.16b, {v17.16b, v18.16b}, v28.16b + + sdot v0.4s, v16.16b, v7.4b[0] + sdot v0.4s, v17.16b, v7.4b[1] + + tbl v16.16b, {v19.16b, v20.16b}, v6.16b + tbl v17.16b, {v20.16b, v21.16b}, v28.16b + + sdot v1.4s, v19.16b, v7.4b[0] + sdot v1.4s, v20.16b, v7.4b[1] + + uzp1 v0.8h, v0.8h, v1.8h + sqrshrun v0.8b, v0.8h, #6 + + subs \h, \h, #2 + fmov x8, d0 + lsr x9, x8, #32 + strh w8, [\dst] + strh w9, [\dst, \d_strd] + add \dst, \dst, \d_strd, lsl #1 + b.gt 2b + + .align JUMP_ALIGN +22: + ldr h18, [\src] + + movi v0.4s, #32, lsl 8 // 64 * 128, bias for SDOT + movi v1.4s, #32, lsl 8 + + sub v18.8b, v18.8b, v5.8b + + tbl v19.16b, {v16.16b, v17.16b}, v6.16b + tbl v20.16b, {v17.16b, v18.16b}, v28.16b + + sdot v0.4s, v16.16b, v7.4b[0] + sdot v0.4s, v17.16b, v7.4b[1] + + sdot v1.4s, v19.16b, v7.4b[0] + sdot v1.4s, v20.16b, v7.4b[1] + + uzp1 v0.8h, v0.8h, v1.8h + sqrshrun v0.8b, v0.8h, #6 + + fmov x8, d0 + lsr x9, x8, #32 + strh w8, [\dst] + strh w9, [\dst, \d_strd] + ret +.endif + + .align JUMP_ALIGN +L(\type\()_8tap_h_hv_\isa): + madd \mx, \mx, w11, w9 + madd w14, \my, w11, w10 // for HV + ldr q28, L(h_tbl_neon_dotprod) + mov w13, 0x2002 // FILTER_WEIGHT * 128 + rounding + sub \src, \src, #4 // src - 4 + dup v27.4s, w13 + ubfx w9, \mx, #7, #7 + and \mx, \mx, #0x7F + ubfx w11, w14, #7, #7 // for HV + and w14, w14, #0x7F // for HV + cmp \w, #4 + csel \mx, \mx, w9, le + add \xmx, x12, \xmx, lsl #3 // subpel H filter address + movi v24.16b, #128 + cbz \my, L(\type\()_8tap_h_\isa) + + // HV cases + cmp \h, #4 + csel w14, w14, w11, le + sub \src, \src, \s_strd, lsl #1 // src - src_stride * 2 - 4 + add \xmy, x12, x14, lsl #3 // subpel V filter address + mov x15, x30 + ldr d7, [\xmy] +.ifc \type, put + ldr q25, L(hv_tbl_neon_dotprod) +.endif + sxtl v7.8h, v7.8b + cmp w10, SHARP1 + b.ne L(\type\()_6tap_hv_\isa) // vertical != SHARP1 + + // HV 8-tap cases + sub \src, \src, \s_strd // src - src_stride * 3 - 4 + cmp \w, #4 + b.eq 40f +.ifc \type, put + b.lt 20f +.endif + + // .align JUMP_ALIGN // fallthrough +80: // HV8 - 8xN+ + ldr q29, L(h_tbl_neon_dotprod) + 16 + ldr q30, L(h_tbl_neon_dotprod) + 32 + ldr d26, [\xmx] +.ifc \type, prep + add \wd_strd, \w, \w +.endif + + .align LOOP_ALIGN +81: + mov \lsrc, \src + mov \ldst, \dst + mov w8, \h + + bl L(\type\()_hv_filter8_\isa) + mov v16.16b, v22.16b + bl L(\type\()_hv_filter8_\isa) + mov v17.16b, v22.16b + bl L(\type\()_hv_filter8_\isa) + mov v18.16b, v22.16b + bl L(\type\()_hv_filter8_\isa) + mov v19.16b, v22.16b + bl L(\type\()_hv_filter8_\isa) + mov v20.16b, v22.16b + bl L(\type\()_hv_filter8_\isa) + mov v21.16b, v22.16b + bl L(\type\()_hv_filter8_\isa) + + .align LOOP_ALIGN +8: + ldr q23, [\lsrc] + add \lsrc, \lsrc, \s_strd + + smull v0.4s, v16.4h, v7.h[0] + smull2 v1.4s, v16.8h, v7.h[0] + mov v16.16b, v17.16b + + sub v23.16b, v23.16b, v24.16b + + mov v5.16b, v27.16b + mov v6.16b, v27.16b + + smlal v0.4s, v17.4h, v7.h[1] + smlal2 v1.4s, v17.8h, v7.h[1] + mov v17.16b, v18.16b + + tbl v2.16b, {v23.16b}, v28.16b + tbl v3.16b, {v23.16b}, v29.16b + tbl v4.16b, {v23.16b}, v30.16b + + smlal v0.4s, v18.4h, v7.h[2] + smlal2 v1.4s, v18.8h, v7.h[2] + mov v18.16b, v19.16b + + sdot v5.4s, v2.16b, v26.4b[0] + sdot v6.4s, v3.16b, v26.4b[0] + + smlal v0.4s, v19.4h, v7.h[3] + smlal2 v1.4s, v19.8h, v7.h[3] + mov v19.16b, v20.16b + + sdot v5.4s, v3.16b, v26.4b[1] + sdot v6.4s, v4.16b, v26.4b[1] + + smlal v0.4s, v20.4h, v7.h[4] + smlal2 v1.4s, v20.8h, v7.h[4] + mov v20.16b, v21.16b + + smlal v0.4s, v21.4h, v7.h[5] + smlal2 v1.4s, v21.8h, v7.h[5] +.ifc \type, prep + uzp1 v23.8h, v5.8h, v6.8h +.endif + mov v21.16b, v22.16b + + smlal v0.4s, v22.4h, v7.h[6] + smlal2 v1.4s, v22.8h, v7.h[6] +.ifc \type, prep + sshr v22.8h, v23.8h, #2 + smlal v0.4s, v22.4h, v7.h[7] + smlal2 v1.4s, v22.8h, v7.h[7] + rshrn v0.4h, v0.4s, #6 + rshrn2 v0.8h, v1.4s, #6 + subs w8, w8, #1 + st1 {v0.8h}, [\ldst], \d_strd + b.gt 8b + add \dst, \dst, #16 +.else + shrn v22.4h, v5.4s, #2 + shrn2 v22.8h, v6.4s, #2 + smlal v0.4s, v22.4h, v7.h[7] + smlal2 v1.4s, v22.8h, v7.h[7] + tbl v0.16b, {v0.16b, v1.16b}, v25.16b + subs w8, w8, #1 + sqrshrun v0.8b, v0.8h, #2 + st1 {v0.8b}, [\ldst], \d_strd + b.gt 8b + add \dst, \dst, #8 +.endif + add \src, \src, #8 + subs \w, \w, #8 + b.gt 81b + ret x15 + + .align JUMP_ALIGN +40: // HV8 - 4xN + ldr s26, [\xmx, #2] + add \src, \src, #2 + + bl L(\type\()_hv_filter4_\isa) + mov v16.16b, v22.16b + bl L(\type\()_hv_filter4_\isa) + mov v17.16b, v22.16b + bl L(\type\()_hv_filter4_\isa) + mov v18.16b, v22.16b + bl L(\type\()_hv_filter4_\isa) + mov v19.16b, v22.16b + bl L(\type\()_hv_filter4_\isa) + mov v20.16b, v22.16b + bl L(\type\()_hv_filter4_\isa) + mov v21.16b, v22.16b + bl L(\type\()_hv_filter4_\isa) + + .align LOOP_ALIGN +4: + ld1 {v4.8b}, [\src], \s_strd + + smull v0.4s, v16.4h, v7.h[0] + smlal v0.4s, v17.4h, v7.h[1] + mov v16.16b, v17.16b + mov v17.16b, v18.16b + sub v4.16b, v4.16b, v24.16b + + smlal v0.4s, v18.4h, v7.h[2] + smlal v0.4s, v19.4h, v7.h[3] + tbl v2.16b, {v4.16b}, v28.16b + mov v5.16b, v27.16b + + mov v18.16b, v19.16b + mov v19.16b, v20.16b + + smlal v0.4s, v20.4h, v7.h[4] + smlal v0.4s, v21.4h, v7.h[5] + + sdot v5.4s, v2.16b, v26.4b[0] + mov v20.16b, v21.16b + mov v21.16b, v22.16b +.ifc \type, put + subs \h, \h, #1 +.endif + smlal v0.4s, v22.4h, v7.h[6] + shrn v22.4h, v5.4s, #2 + + smlal v0.4s, v22.4h, v7.h[7] +.ifc \type, prep + rshrn v0.4h, v0.4s, #6 + str d0, [\dst], #8 + subs \h, \h, #1 +.else + tbl v0.16b, {v0.16b, v1.16b}, v25.16b + sqrshrun v0.8b, v0.8h, #2 + str s0, [\dst] + add \dst, \dst, \d_strd +.endif + b.gt 4b + ret x15 + +.ifc \type, put + .align JUMP_ALIGN +20: // HV8 - 2xN + ldr s26, [\xmx, #2] + add \src, \src, #2 + + bl L(\type\()_hv_filter4_\isa) + mov v16.16b, v22.16b + bl L(\type\()_hv_filter4_\isa) + mov v17.16b, v22.16b + bl L(\type\()_hv_filter4_\isa) + mov v18.16b, v22.16b + bl L(\type\()_hv_filter4_\isa) + mov v19.16b, v22.16b + bl L(\type\()_hv_filter4_\isa) + mov v20.16b, v22.16b + bl L(\type\()_hv_filter4_\isa) + mov v21.16b, v22.16b + bl L(\type\()_hv_filter4_\isa) + + .align LOOP_ALIGN +2: + ld1 {v4.8b}, [\src], \s_strd + + smull v0.4s, v16.4h, v7.h[0] + smlal v0.4s, v17.4h, v7.h[1] + mov v16.16b, v17.16b + mov v17.16b, v18.16b + sub v4.16b, v4.16b, v24.16b + + smlal v0.4s, v18.4h, v7.h[2] + smlal v0.4s, v19.4h, v7.h[3] + tbl v2.16b, {v4.16b}, v28.16b + mov v5.16b, v27.16b + + mov v18.16b, v19.16b + mov v19.16b, v20.16b + + smlal v0.4s, v20.4h, v7.h[4] + smlal v0.4s, v21.4h, v7.h[5] + + sdot v5.4s, v2.16b, v26.4b[0] + mov v20.16b, v21.16b + mov v21.16b, v22.16b + + subs \h, \h, #1 + smlal v0.4s, v22.4h, v7.h[6] + shrn v22.4h, v5.4s, #2 + + smlal v0.4s, v22.4h, v7.h[7] + tbl v0.16b, {v0.16b, v1.16b}, v25.16b + sqrshrun v0.8b, v0.8h, #2 + + str h0, [\dst] + add \dst, \dst, \d_strd + b.gt 2b + ret x15 +.endif + + .align JUMP_ALIGN +L(\type\()_6tap_hv_\isa): + cmp \w, #4 + b.eq 40f +.ifc \type, put + b.lt 20f +.endif + + // .align JUMP_ALIGN // fallthrough +80: // HV6 - 8xN+ + ldr q29, L(h_tbl_neon_dotprod) + 16 + ldr q30, L(h_tbl_neon_dotprod) + 32 + ldr d26, [\xmx] +.ifc \type, prep + add \wd_strd, \w, \w +.endif + + .align LOOP_ALIGN +81: + mov \lsrc, \src + mov \ldst, \dst + mov w8, \h + + bl L(\type\()_hv_filter8_\isa) + mov v16.16b, v22.16b + bl L(\type\()_hv_filter8_\isa) + mov v17.16b, v22.16b + bl L(\type\()_hv_filter8_\isa) + mov v18.16b, v22.16b + bl L(\type\()_hv_filter8_\isa) + mov v19.16b, v22.16b + bl L(\type\()_hv_filter8_\isa) + mov v20.16b, v22.16b + + .align LOOP_ALIGN +8: + ldr q23, [\xmy] + add \xmy, \xmy, \s_strd + + smull v0.4s, v16.4h, v7.h[1] + smull2 v1.4s, v16.8h, v7.h[1] + sub v23.16b, v23.16b, v24.16b + mov v16.16b, v17.16b + + mov v5.16b, v27.16b + mov v6.16b, v27.16b + + tbl v2.16b, {v23.16b}, v28.16b + tbl v3.16b, {v23.16b}, v29.16b + + smlal v0.4s, v17.4h, v7.h[2] + smlal2 v1.4s, v17.8h, v7.h[2] + tbl v4.16b, {v23.16b}, v30.16b + mov v17.16b, v18.16b + + sdot v5.4s, v2.16b, v26.4b[0] + sdot v6.4s, v3.16b, v26.4b[0] + smlal v0.4s, v18.4h, v7.h[3] + smlal2 v1.4s, v18.8h, v7.h[3] + mov v18.16b, v19.16b + + sdot v5.4s, v3.16b, v26.4b[1] + sdot v6.4s, v4.16b, v26.4b[1] + smlal v0.4s, v19.4h, v7.h[4] + smlal2 v1.4s, v19.8h, v7.h[4] + mov v19.16b, v20.16b + uzp1 v23.8h, v5.8h, v6.8h + + smlal v0.4s, v20.4h, v7.h[5] + smlal2 v1.4s, v20.8h, v7.h[5] + sshr v20.8h, v23.8h, #2 +.ifc \type, prep + smlal v0.4s, v20.4h, v7.h[6] + smlal2 v1.4s, v20.8h, v7.h[6] + rshrn v0.4h, v0.4s, #6 + rshrn2 v0.8h, v1.4s, #6 + st1 {v0.8h}, [\ldst], \d_strd + subs w8, w8, #1 + b.gt 8b + add \dst, \dst, #16 +.else + subs w8, w8, #1 + smlal v0.4s, v20.4h, v7.h[6] + smlal2 v1.4s, v20.8h, v7.h[6] + tbl v0.16b, {v0.16b, v1.16b}, v25.16b + sqrshrun v0.8b, v0.8h, #2 + st1 {v0.8b}, [\ldst], \d_strd + b.gt 8b + add \dst, \dst, #8 +.endif + add \src, \src, #8 + subs \w, \w, #8 + b.gt 81b + ret x15 + + .align FUNC_ALIGN +L(\type\()_hv_filter8_\isa): + ldr q4, [\lsrc] + add \lsrc, \lsrc, \s_strd + sub v4.16b, v4.16b, v24.16b + mov v22.16b, v27.16b + mov v23.16b, v27.16b + tbl v2.16b, {v4.16b}, v28.16b + tbl v3.16b, {v4.16b}, v29.16b + tbl v4.16b, {v4.16b}, v30.16b + sdot v22.4s, v2.16b, v26.4b[0] + sdot v22.4s, v3.16b, v26.4b[1] + sdot v23.4s, v3.16b, v26.4b[0] + sdot v23.4s, v4.16b, v26.4b[1] + shrn v22.4h, v22.4s, #2 + shrn2 v22.8h, v23.4s, #2 + ret + + .align FUNC_ALIGN +L(\type\()_hv_filter4_\isa): + mov v22.16b, v27.16b + ld1 {v4.8b}, [\src], \s_strd + sub v4.16b, v4.16b, v24.16b + tbl v2.16b, {v4.16b}, v28.16b + sdot v22.4s, v2.16b, v26.4b[0] + shrn v22.4h, v22.4s, #2 + ret + + .align JUMP_ALIGN +40: // HV6 - 4xN + ldr s26, [\xmx, #2] + add \src, \src, #2 + + bl L(\type\()_hv_filter4_\isa) + mov v16.16b, v22.16b + bl L(\type\()_hv_filter4_\isa) + mov v17.16b, v22.16b + bl L(\type\()_hv_filter4_\isa) + mov v18.16b, v22.16b + bl L(\type\()_hv_filter4_\isa) + mov v19.16b, v22.16b + bl L(\type\()_hv_filter4_\isa) + mov v20.16b, v22.16b + + .align LOOP_ALIGN +4: + ld1 {v4.8b}, [\src], \s_strd + + smull v0.4s, v16.4h, v7.h[1] + smlal v0.4s, v17.4h, v7.h[2] + sub v4.16b, v4.16b, v24.16b + mov v16.16b, v17.16b + mov v17.16b, v18.16b + + smlal v0.4s, v18.4h, v7.h[3] + smlal v0.4s, v19.4h, v7.h[4] + tbl v2.16b, {v4.16b}, v28.16b + mov v5.16b, v27.16b + + mov v18.16b, v19.16b + mov v19.16b, v20.16b + sdot v5.4s, v2.16b, v26.4b[0] + + smlal v0.4s, v20.4h, v7.h[5] + shrn v20.4h, v5.4s, #2 +.ifc \type, prep + smlal v0.4s, v20.4h, v7.h[6] + rshrn v0.4h, v0.4s, #6 + str d0, [\dst], #8 + subs \h, \h, #1 +.else + subs \h, \h, #1 + smlal v0.4s, v20.4h, v7.h[6] + tbl v0.16b, {v0.16b}, v25.16b + sqrshrun v0.8b, v0.8h, #2 + str s0, [\dst] + add \dst, \dst, \d_strd +.endif + b.gt 4b + ret x15 + +.ifc \type, put + .align JUMP_ALIGN +20: // HV6 - 2xN + ldr s26, [\xmx, #2] + add \src, \src, #2 + + bl L(\type\()_hv_filter4_\isa) + mov v16.16b, v22.16b + bl L(\type\()_hv_filter4_\isa) + mov v17.16b, v22.16b + bl L(\type\()_hv_filter4_\isa) + mov v18.16b, v22.16b + bl L(\type\()_hv_filter4_\isa) + mov v19.16b, v22.16b + bl L(\type\()_hv_filter4_\isa) + mov v20.16b, v22.16b + + .align LOOP_ALIGN +2: + ld1 {v4.8b}, [\src], \s_strd + + smull v0.4s, v16.4h, v7.h[1] + smlal v0.4s, v17.4h, v7.h[2] + sub v4.16b, v4.16b, v24.16b + mov v16.16b, v17.16b + mov v17.16b, v18.16b + + smlal v0.4s, v18.4h, v7.h[3] + smlal v0.4s, v19.4h, v7.h[4] + tbl v2.16b, {v4.16b}, v28.16b + mov v5.16b, v27.16b + + mov v18.16b, v19.16b + mov v19.16b, v20.16b + sdot v5.4s, v2.16b, v26.4b[0] + + smlal v0.4s, v20.4h, v7.h[5] + shrn v20.4h, v5.4s, #2 + + subs \h, \h, #1 + smlal v0.4s, v20.4h, v7.h[6] + + tbl v0.16b, {v0.16b}, v25.16b + sqrshrun v0.8b, v0.8h, #2 + + str h0, [\dst] + add \dst, \dst, \d_strd + b.gt 2b + ret x15 +.endif + + .align JUMP_ALIGN +L(\type\()_8tap_h_\isa): + adr x9, L(\type\()_8tap_h_\isa\()_tbl) + ldrh w8, [x9, x8, lsl #1] +.ifc \type, put + mov w10, #0x2022 // 64 * 128 + 34, bias and rounding for SDOT + dup v27.4s, w10 +.endif + sub x9, x9, x8 + br x9 + +.ifc \type, put + .align JUMP_ALIGN +20: // H - 2xN + AARCH64_VALID_JUMP_TARGET + add \src, \src, #2 + ldr s6, [\xmx, #2] + + .align LOOP_ALIGN +2: + ldr d0, [\src] + ldr d1, [\src, \s_strd] + add \src, \src, \s_strd, lsl #1 + + sub v0.8b, v0.8b, v24.8b + sub v1.8b, v1.8b, v24.8b + + mov v4.16b, v27.16b + mov v5.16b, v27.16b + + tbl v2.16b, {v0.16b}, v28.16b + tbl v3.16b, {v1.16b}, v28.16b + + sdot v4.4s, v2.16b, v6.4b[0] + sdot v5.4s, v3.16b, v6.4b[0] + + uzp1 v4.8h, v4.8h, v5.8h + sqshrun v4.8b, v4.8h, #6 + + subs \h, \h, #2 + fmov x8, d4 + lsr x9, x8, #32 + strh w8, [\dst] + strh w9, [\dst, \d_strd] + add \dst, \dst, \d_strd, lsl #1 + b.gt 2b + ret + +.endif + + .align JUMP_ALIGN +40: // H - 4xN + AARCH64_VALID_JUMP_TARGET + add \src, \src, #2 + ldr s26, [\xmx, #2] + + .align LOOP_ALIGN +4: + ldr d0, [\src] + ldr d1, [\src, \s_strd] + add \src, \src, \s_strd, lsl #1 + + sub v0.8b, v0.8b, v24.8b + sub v1.8b, v1.8b, v24.8b + + mov v4.16b, v27.16b + mov v5.16b, v27.16b + + tbl v2.16b, {v0.16b}, v28.16b + tbl v3.16b, {v1.16b}, v28.16b + + sdot v4.4s, v2.16b, v26.4b[0] + sdot v5.4s, v3.16b, v26.4b[0] +.ifc \type, prep + subs \h, \h, #2 + shrn v4.4h, v4.4s, #2 + shrn2 v4.8h, v5.4s, #2 + str q4, [\dst], #16 +.else + uzp1 v4.8h, v4.8h, v5.8h + sqshrun v4.8b, v4.8h, #6 + subs \h, \h, #2 + fmov x8, d4 + lsr x9, x8, #32 + str w8, [\dst] + str w9, [\dst, \d_strd] + add \dst, \dst, \d_strd, lsl #1 +.endif + b.gt 4b + ret + + .align JUMP_ALIGN +80: // H - 8xN + AARCH64_VALID_JUMP_TARGET + ldr q29, L(h_tbl_neon_dotprod) + 16 + ldr q30, L(h_tbl_neon_dotprod) + 32 + ldr d26, [\xmx] + + .align LOOP_ALIGN +8: + ldr q0, [\src] + ldr q16, [\src, \s_strd] + add \src, \src, \s_strd, lsl #1 + + sub v0.16b, v0.16b, v24.16b + sub v16.16b, v16.16b, v24.16b + + mov v4.16b, v27.16b + mov v5.16b, v27.16b + mov v20.16b, v27.16b + mov v21.16b, v27.16b + + tbl v1.16b, {v0.16b}, v28.16b + tbl v2.16b, {v0.16b}, v29.16b + tbl v3.16b, {v0.16b}, v30.16b + tbl v17.16b, {v16.16b}, v28.16b + tbl v18.16b, {v16.16b}, v29.16b + tbl v19.16b, {v16.16b}, v30.16b + + sdot v4.4s, v1.16b, v26.4b[0] + sdot v5.4s, v2.16b, v26.4b[0] + sdot v20.4s, v17.16b, v26.4b[0] + sdot v21.4s, v18.16b, v26.4b[0] + sdot v4.4s, v2.16b, v26.4b[1] + sdot v5.4s, v3.16b, v26.4b[1] + sdot v20.4s, v18.16b, v26.4b[1] + sdot v21.4s, v19.16b, v26.4b[1] + + uzp1 v4.8h, v4.8h, v5.8h + uzp1 v20.8h, v20.8h, v21.8h +.ifc \type, prep + sshr v4.8h, v4.8h, #2 + sshr v20.8h, v20.8h, #2 + subs \h, \h, #2 + stp q4, q20, [\dst], #32 +.else + sqshrun v4.8b, v4.8h, #6 + sqshrun v20.8b, v20.8h, #6 + subs \h, \h, #2 + str d4, [\dst] + str d20, [\dst, \d_strd] + add \dst, \dst, \d_strd, lsl #1 +.endif + b.gt 8b + ret + + .align JUMP_ALIGN +160: // H - 16xN + AARCH64_VALID_JUMP_TARGET + ldr q29, L(h_tbl_neon_dotprod) + 16 + ldr q30, L(h_tbl_neon_dotprod) + 32 + ldr q31, L(h_tbl_neon_dotprod) + 48 + ldr d26, [\xmx] + + .align LOOP_ALIGN +16: + ldp q16, q17, [\src] + add \src, \src, \s_strd + + sub v16.16b, v16.16b, v24.16b + sub v17.16b, v17.16b, v24.16b + + mov v6.16b, v27.16b + mov v7.16b, v27.16b + mov v22.16b, v27.16b + mov v23.16b, v27.16b + + tbl v0.16b, {v16.16b}, v28.16b + tbl v1.16b, {v16.16b}, v29.16b + tbl v2.16b, {v16.16b}, v30.16b + tbl v3.16b, {v16.16b, v17.16b}, v31.16b + tbl v4.16b, {v17.16b}, v28.16b + + sdot v6.4s, v0.16b, v26.4b[0] + sdot v7.4s, v1.16b, v26.4b[0] + sdot v22.4s, v2.16b, v26.4b[0] + sdot v23.4s, v3.16b, v26.4b[0] + sdot v6.4s, v1.16b, v26.4b[1] + sdot v7.4s, v2.16b, v26.4b[1] + sdot v22.4s, v3.16b, v26.4b[1] + sdot v23.4s, v4.16b, v26.4b[1] + + uzp1 v6.8h, v6.8h, v7.8h + uzp1 v22.8h, v22.8h, v23.8h +.ifc \type, prep + sshr v6.8h, v6.8h, #2 + sshr v22.8h, v22.8h, #2 + subs \h, \h, #1 + stp q6, q22, [\dst], #32 +.else + sqshrun v6.8b, v6.8h, #6 + sqshrun2 v6.16b, v22.8h, #6 + subs \h, \h, #1 + str q6, [\dst] + add \dst, \dst, \d_strd +.endif + b.gt 16b + ret + + .align JUMP_ALIGN +320: // H - 32xN+ +640: +1280: + AARCH64_VALID_JUMP_TARGET + ldr q29, L(h_tbl_neon_dotprod) + 16 + ldr q30, L(h_tbl_neon_dotprod) + 32 + ldr q31, L(h_tbl_neon_dotprod) + 48 + ldr d26, [\xmx] +.ifc \type, put + sub \d_strd, \d_strd, \w, uxtw +.endif + sub \s_strd, \s_strd, \w, uxtw + mov w8, \w + + .align LOOP_ALIGN +32: + ldp q16, q17, [\src], #16 + + sub v16.16b, v16.16b, v24.16b + sub v17.16b, v17.16b, v24.16b + + mov v6.16b, v27.16b + mov v7.16b, v27.16b + mov v22.16b, v27.16b + mov v23.16b, v27.16b + + tbl v0.16b, {v16.16b}, v28.16b + tbl v1.16b, {v16.16b}, v29.16b + tbl v2.16b, {v16.16b}, v30.16b + tbl v3.16b, {v16.16b, v17.16b}, v31.16b + tbl v4.16b, {v17.16b}, v28.16b + + sdot v6.4s, v0.16b, v26.4b[0] + sdot v7.4s, v1.16b, v26.4b[0] + sdot v22.4s, v2.16b, v26.4b[0] + sdot v23.4s, v3.16b, v26.4b[0] + sdot v6.4s, v1.16b, v26.4b[1] + sdot v7.4s, v2.16b, v26.4b[1] + sdot v22.4s, v3.16b, v26.4b[1] + sdot v23.4s, v4.16b, v26.4b[1] + + uzp1 v6.8h, v6.8h, v7.8h + uzp1 v22.8h, v22.8h, v23.8h +.ifc \type, prep + sshr v6.8h, v6.8h, #2 + sshr v22.8h, v22.8h, #2 + subs w8, w8, #16 + stp q6, q22, [\dst], #32 +.else + sqshrun v6.8b, v6.8h, #6 + sqshrun2 v6.16b, v22.8h, #6 + subs w8, w8, #16 + str q6, [\dst], #16 +.endif + b.gt 32b + + add \src, \src, \s_strd +.ifc \type, put + add \dst, \dst, \d_strd +.endif + mov w8, \w + subs \h, \h, #1 + b.gt 32b + ret + +L(\type\()_8tap_h_\isa\()_tbl): + .hword (L(\type\()_8tap_h_\isa\()_tbl) - 1280b) + .hword (L(\type\()_8tap_h_\isa\()_tbl) - 640b) + .hword (L(\type\()_8tap_h_\isa\()_tbl) - 320b) + .hword (L(\type\()_8tap_h_\isa\()_tbl) - 160b) + .hword (L(\type\()_8tap_h_\isa\()_tbl) - 80b) + .hword (L(\type\()_8tap_h_\isa\()_tbl) - 40b) +.ifc \type, put + .hword (L(\type\()_8tap_h_\isa\()_tbl) - 20b) +.endif +endfunc +.endm + +// dst(x0), d_strd(x7), src(x1), s_strd(x2), w(w3), h(w4), mx(w5), my(w6) +// xmx(x5), xmy(x6), ldst(x5), lsrc(x6), wd_strd(w7) +filter_8tap_fn prep, neon_dotprod, x0, x7, x1, x2, w3, w4, w5, w6, x5, x6, x5, x6, w7 + +// dst(x0) d_strd(x1) src(x2) s_strd(x3) w(w4) h(w5) mx(w6) my(w7) +// xmx(x6), xmy(x7), ldst(x6), lsrc(x7), wd_strd(w1) +filter_8tap_fn put, neon_dotprod, x0, x1, x2, x3, w4, w5, w6, w7, x6, x7, x6, x7, w1 + +DISABLE_DOTPROD +#endif // HAVE_DOTPROD diff --git a/third_party/dav1d/src/arm/64/msac.S b/third_party/dav1d/src/arm/64/msac.S index 7bef9243fb..9033072a82 100644 --- a/third_party/dav1d/src/arm/64/msac.S +++ b/third_party/dav1d/src/arm/64/msac.S @@ -288,10 +288,8 @@ function msac_decode_hi_tok_neon, export=1 mvni v30.4h, #0x3f // 0xffc0 ldrh w9, [x1, #6] // count = cdf[n_symbols] ld1r {v3.4h}, [x16] // rng - movrel x16, bits ld1 {v29.4h}, [x17] // EC_MIN_PROB * (n_symbols - ret) add x17, x0, #DIF + 6 - ld1 {v16.8h}, [x16] mov w13, #-24 and v17.8b, v0.8b, v30.8b // cdf & 0xffc0 ldr w10, [x0, #ALLOW_UPDATE_CDF] @@ -305,30 +303,27 @@ function msac_decode_hi_tok_neon, export=1 add v4.4h, v17.4h, v29.4h // v = cdf + EC_MIN_PROB * (n_symbols - ret) add v4.4h, v6.4h, v4.4h // v = ((cdf >> EC_PROB_SHIFT) * r) >> 1 + EC_MIN_PROB * (n_symbols - ret) str h3, [sp, #14] // store original u = s->rng - cmhs v2.8h, v1.8h, v4.8h // c >= v + cmhs v2.4h, v1.4h, v4.4h // c >= v str q4, [sp, #16] // store v values to allow indexed access - and v6.16b, v2.16b, v16.16b // One bit per halfword set in the mask - addv h6, v6.8h // Aggregate mask bits - umov w3, v6.h[0] + addv h6, v2.4h // -4 + ret add w13, w13, #5 - rbit w3, w3 + smov w15, v6.h[0] add x8, sp, #16 - clz w15, w3 // ret + add w15, w15, #4 // ret cbz w10, 2f // update_cdf - movi v5.8b, #0xff + sub v5.4h, v0.4h, v2.4h // cdf[i] + (i >= val ? 1 : 0) mov w4, #-5 - urhadd v4.4h, v5.4h, v2.4h // i >= val ? -1 : 32768 + orr v2.4h, #0x80, lsl #8 // i >= val ? -1 : 32768 sub w4, w4, w9, lsr #4 // -((count >> 4) + 5) - sub v4.4h, v4.4h, v0.4h // (32768 - cdf[i]) or (-1 - cdf[i]) + sub v4.4h, v2.4h, v0.4h // (32768 - cdf[i]) or (-1 - cdf[i]) dup v6.4h, w4 // -rate sub w9, w9, w9, lsr #5 // count - (count == 32) - sub v0.4h, v0.4h, v2.4h // cdf + (i >= val ? 1 : 0) sshl v4.4h, v4.4h, v6.4h // ({32768,-1} - cdf[i]) >> rate add w9, w9, #1 // count + (count < 32) - add v0.4h, v0.4h, v4.4h // cdf + (32768 - cdf[i]) >> rate + add v0.4h, v5.4h, v4.4h // cdf[i] + (32768 - cdf[i]) >> rate st1 {v0.4h}, [x1] and v17.8b, v0.8b, v30.8b // cdf & 0xffc0 strh w9, [x1, #6] diff --git a/third_party/dav1d/src/arm/itx.h b/third_party/dav1d/src/arm/itx.h index 17234e027a..2a58a31322 100644 --- a/third_party/dav1d/src/arm/itx.h +++ b/third_party/dav1d/src/arm/itx.h @@ -28,34 +28,6 @@ #include "src/cpu.h" #include "src/itx.h" -#define decl_itx2_fns(w, h, opt) \ -decl_itx_fn(BF(dav1d_inv_txfm_add_dct_dct_##w##x##h, opt)); \ -decl_itx_fn(BF(dav1d_inv_txfm_add_identity_identity_##w##x##h, opt)) - -#define decl_itx12_fns(w, h, opt) \ -decl_itx2_fns(w, h, opt); \ -decl_itx_fn(BF(dav1d_inv_txfm_add_dct_adst_##w##x##h, opt)); \ -decl_itx_fn(BF(dav1d_inv_txfm_add_dct_flipadst_##w##x##h, opt)); \ -decl_itx_fn(BF(dav1d_inv_txfm_add_dct_identity_##w##x##h, opt)); \ -decl_itx_fn(BF(dav1d_inv_txfm_add_adst_dct_##w##x##h, opt)); \ -decl_itx_fn(BF(dav1d_inv_txfm_add_adst_adst_##w##x##h, opt)); \ -decl_itx_fn(BF(dav1d_inv_txfm_add_adst_flipadst_##w##x##h, opt)); \ -decl_itx_fn(BF(dav1d_inv_txfm_add_flipadst_dct_##w##x##h, opt)); \ -decl_itx_fn(BF(dav1d_inv_txfm_add_flipadst_adst_##w##x##h, opt)); \ -decl_itx_fn(BF(dav1d_inv_txfm_add_flipadst_flipadst_##w##x##h, opt)); \ -decl_itx_fn(BF(dav1d_inv_txfm_add_identity_dct_##w##x##h, opt)) - -#define decl_itx16_fns(w, h, opt) \ -decl_itx12_fns(w, h, opt); \ -decl_itx_fn(BF(dav1d_inv_txfm_add_adst_identity_##w##x##h, opt)); \ -decl_itx_fn(BF(dav1d_inv_txfm_add_flipadst_identity_##w##x##h, opt)); \ -decl_itx_fn(BF(dav1d_inv_txfm_add_identity_adst_##w##x##h, opt)); \ -decl_itx_fn(BF(dav1d_inv_txfm_add_identity_flipadst_##w##x##h, opt)) - -#define decl_itx17_fns(w, h, opt) \ -decl_itx16_fns(w, h, opt); \ -decl_itx_fn(BF(dav1d_inv_txfm_add_wht_wht_##w##x##h, opt)) - decl_itx17_fns( 4, 4, neon); decl_itx16_fns( 4, 8, neon); decl_itx16_fns( 4, 16, neon); @@ -78,41 +50,6 @@ decl_itx_fn(BF(dav1d_inv_txfm_add_dct_dct_64x32, neon)); decl_itx_fn(BF(dav1d_inv_txfm_add_dct_dct_64x64, neon)); static ALWAYS_INLINE void itx_dsp_init_arm(Dav1dInvTxfmDSPContext *const c, int bpc) { -#define assign_itx_fn(pfx, w, h, type, type_enum, ext) \ - c->itxfm_add[pfx##TX_##w##X##h][type_enum] = \ - BF(dav1d_inv_txfm_add_##type##_##w##x##h, ext) - -#define assign_itx1_fn(pfx, w, h, ext) \ - assign_itx_fn(pfx, w, h, dct_dct, DCT_DCT, ext) - -#define assign_itx2_fn(pfx, w, h, ext) \ - assign_itx1_fn(pfx, w, h, ext); \ - assign_itx_fn(pfx, w, h, identity_identity, IDTX, ext) - -#define assign_itx12_fn(pfx, w, h, ext) \ - assign_itx2_fn(pfx, w, h, ext); \ - assign_itx_fn(pfx, w, h, dct_adst, ADST_DCT, ext); \ - assign_itx_fn(pfx, w, h, dct_flipadst, FLIPADST_DCT, ext); \ - assign_itx_fn(pfx, w, h, dct_identity, H_DCT, ext); \ - assign_itx_fn(pfx, w, h, adst_dct, DCT_ADST, ext); \ - assign_itx_fn(pfx, w, h, adst_adst, ADST_ADST, ext); \ - assign_itx_fn(pfx, w, h, adst_flipadst, FLIPADST_ADST, ext); \ - assign_itx_fn(pfx, w, h, flipadst_dct, DCT_FLIPADST, ext); \ - assign_itx_fn(pfx, w, h, flipadst_adst, ADST_FLIPADST, ext); \ - assign_itx_fn(pfx, w, h, flipadst_flipadst, FLIPADST_FLIPADST, ext); \ - assign_itx_fn(pfx, w, h, identity_dct, V_DCT, ext) - -#define assign_itx16_fn(pfx, w, h, ext) \ - assign_itx12_fn(pfx, w, h, ext); \ - assign_itx_fn(pfx, w, h, adst_identity, H_ADST, ext); \ - assign_itx_fn(pfx, w, h, flipadst_identity, H_FLIPADST, ext); \ - assign_itx_fn(pfx, w, h, identity_adst, V_ADST, ext); \ - assign_itx_fn(pfx, w, h, identity_flipadst, V_FLIPADST, ext) - -#define assign_itx17_fn(pfx, w, h, ext) \ - assign_itx16_fn(pfx, w, h, ext); \ - assign_itx_fn(pfx, w, h, wht_wht, WHT_WHT, ext) - const unsigned flags = dav1d_get_cpu_flags(); if (!(flags & DAV1D_ARM_CPU_FLAG_NEON)) return; diff --git a/third_party/dav1d/src/arm/mc.h b/third_party/dav1d/src/arm/mc.h index 06cd533a9b..7e57fd37cb 100644 --- a/third_party/dav1d/src/arm/mc.h +++ b/third_party/dav1d/src/arm/mc.h @@ -30,26 +30,40 @@ #include "src/mc.h" #include "src/cpu.h" -decl_mc_fn(BF(dav1d_put_8tap_regular, neon)); -decl_mc_fn(BF(dav1d_put_8tap_regular_smooth, neon)); -decl_mc_fn(BF(dav1d_put_8tap_regular_sharp, neon)); -decl_mc_fn(BF(dav1d_put_8tap_smooth, neon)); -decl_mc_fn(BF(dav1d_put_8tap_smooth_regular, neon)); -decl_mc_fn(BF(dav1d_put_8tap_smooth_sharp, neon)); -decl_mc_fn(BF(dav1d_put_8tap_sharp, neon)); -decl_mc_fn(BF(dav1d_put_8tap_sharp_regular, neon)); -decl_mc_fn(BF(dav1d_put_8tap_sharp_smooth, neon)); -decl_mc_fn(BF(dav1d_put_bilin, neon)); +#define decl_8tap_gen(decl_name, fn_name, opt) \ + decl_##decl_name##_fn(BF(dav1d_##fn_name##_8tap_regular, opt)); \ + decl_##decl_name##_fn(BF(dav1d_##fn_name##_8tap_regular_smooth, opt)); \ + decl_##decl_name##_fn(BF(dav1d_##fn_name##_8tap_regular_sharp, opt)); \ + decl_##decl_name##_fn(BF(dav1d_##fn_name##_8tap_smooth_regular, opt)); \ + decl_##decl_name##_fn(BF(dav1d_##fn_name##_8tap_smooth, opt)); \ + decl_##decl_name##_fn(BF(dav1d_##fn_name##_8tap_smooth_sharp, opt)); \ + decl_##decl_name##_fn(BF(dav1d_##fn_name##_8tap_sharp_regular, opt)); \ + decl_##decl_name##_fn(BF(dav1d_##fn_name##_8tap_sharp_smooth, opt)); \ + decl_##decl_name##_fn(BF(dav1d_##fn_name##_8tap_sharp, opt)) + +#define decl_8tap_fns(opt) \ + decl_8tap_gen(mc, put, opt); \ + decl_8tap_gen(mct, prep, opt) + +#define init_8tap_gen(name, opt) \ + init_##name##_fn(FILTER_2D_8TAP_REGULAR, 8tap_regular, opt); \ + init_##name##_fn(FILTER_2D_8TAP_REGULAR_SMOOTH, 8tap_regular_smooth, opt); \ + init_##name##_fn(FILTER_2D_8TAP_REGULAR_SHARP, 8tap_regular_sharp, opt); \ + init_##name##_fn(FILTER_2D_8TAP_SMOOTH_REGULAR, 8tap_smooth_regular, opt); \ + init_##name##_fn(FILTER_2D_8TAP_SMOOTH, 8tap_smooth, opt); \ + init_##name##_fn(FILTER_2D_8TAP_SMOOTH_SHARP, 8tap_smooth_sharp, opt); \ + init_##name##_fn(FILTER_2D_8TAP_SHARP_REGULAR, 8tap_sharp_regular, opt); \ + init_##name##_fn(FILTER_2D_8TAP_SHARP_SMOOTH, 8tap_sharp_smooth, opt); \ + init_##name##_fn(FILTER_2D_8TAP_SHARP, 8tap_sharp, opt) + +#define init_8tap_fns(opt) \ + init_8tap_gen(mc, opt); \ + init_8tap_gen(mct, opt) + +decl_8tap_fns(neon); +decl_8tap_fns(neon_dotprod); -decl_mct_fn(BF(dav1d_prep_8tap_regular, neon)); -decl_mct_fn(BF(dav1d_prep_8tap_regular_smooth, neon)); -decl_mct_fn(BF(dav1d_prep_8tap_regular_sharp, neon)); -decl_mct_fn(BF(dav1d_prep_8tap_smooth, neon)); -decl_mct_fn(BF(dav1d_prep_8tap_smooth_regular, neon)); -decl_mct_fn(BF(dav1d_prep_8tap_smooth_sharp, neon)); -decl_mct_fn(BF(dav1d_prep_8tap_sharp, neon)); -decl_mct_fn(BF(dav1d_prep_8tap_sharp_regular, neon)); -decl_mct_fn(BF(dav1d_prep_8tap_sharp_smooth, neon)); +decl_mc_fn(BF(dav1d_put_bilin, neon)); decl_mct_fn(BF(dav1d_prep_bilin, neon)); decl_avg_fn(BF(dav1d_avg, neon)); @@ -77,27 +91,10 @@ static ALWAYS_INLINE void mc_dsp_init_arm(Dav1dMCDSPContext *const c) { if (!(flags & DAV1D_ARM_CPU_FLAG_NEON)) return; - init_mc_fn (FILTER_2D_8TAP_REGULAR, 8tap_regular, neon); - init_mc_fn (FILTER_2D_8TAP_REGULAR_SMOOTH, 8tap_regular_smooth, neon); - init_mc_fn (FILTER_2D_8TAP_REGULAR_SHARP, 8tap_regular_sharp, neon); - init_mc_fn (FILTER_2D_8TAP_SMOOTH_REGULAR, 8tap_smooth_regular, neon); - init_mc_fn (FILTER_2D_8TAP_SMOOTH, 8tap_smooth, neon); - init_mc_fn (FILTER_2D_8TAP_SMOOTH_SHARP, 8tap_smooth_sharp, neon); - init_mc_fn (FILTER_2D_8TAP_SHARP_REGULAR, 8tap_sharp_regular, neon); - init_mc_fn (FILTER_2D_8TAP_SHARP_SMOOTH, 8tap_sharp_smooth, neon); - init_mc_fn (FILTER_2D_8TAP_SHARP, 8tap_sharp, neon); - init_mc_fn (FILTER_2D_BILINEAR, bilin, neon); - - init_mct_fn(FILTER_2D_8TAP_REGULAR, 8tap_regular, neon); - init_mct_fn(FILTER_2D_8TAP_REGULAR_SMOOTH, 8tap_regular_smooth, neon); - init_mct_fn(FILTER_2D_8TAP_REGULAR_SHARP, 8tap_regular_sharp, neon); - init_mct_fn(FILTER_2D_8TAP_SMOOTH_REGULAR, 8tap_smooth_regular, neon); - init_mct_fn(FILTER_2D_8TAP_SMOOTH, 8tap_smooth, neon); - init_mct_fn(FILTER_2D_8TAP_SMOOTH_SHARP, 8tap_smooth_sharp, neon); - init_mct_fn(FILTER_2D_8TAP_SHARP_REGULAR, 8tap_sharp_regular, neon); - init_mct_fn(FILTER_2D_8TAP_SHARP_SMOOTH, 8tap_sharp_smooth, neon); - init_mct_fn(FILTER_2D_8TAP_SHARP, 8tap_sharp, neon); - init_mct_fn(FILTER_2D_BILINEAR, bilin, neon); + init_8tap_fns(neon); + + init_mc_fn (FILTER_2D_BILINEAR, bilin, neon); + init_mct_fn(FILTER_2D_BILINEAR, bilin, neon); c->avg = BF(dav1d_avg, neon); c->w_avg = BF(dav1d_w_avg, neon); @@ -111,4 +108,12 @@ static ALWAYS_INLINE void mc_dsp_init_arm(Dav1dMCDSPContext *const c) { c->warp8x8 = BF(dav1d_warp_affine_8x8, neon); c->warp8x8t = BF(dav1d_warp_affine_8x8t, neon); c->emu_edge = BF(dav1d_emu_edge, neon); + +#if ARCH_AARCH64 +#if HAVE_DOTPROD && BITDEPTH == 8 + if (!(flags & DAV1D_ARM_CPU_FLAG_DOTPROD)) return; + + init_8tap_fns(neon_dotprod); +#endif // HAVE_DOTPROD && BITDEPTH == 8 +#endif // ARCH_AARCH64 } diff --git a/third_party/dav1d/src/cdf.c b/third_party/dav1d/src/cdf.c index e0f2132e00..d9721dad46 100644 --- a/third_party/dav1d/src/cdf.c +++ b/third_party/dav1d/src/cdf.c @@ -65,631 +65,638 @@ #define CDF15(a,b,c,d,e,f,g,h,i,j,k,l,m,n,o) \ CDF1(a), CDF14(b,c,d,e,f,g,h,i,j,k,l,m,n,o) -static const CdfModeContext av1_default_cdf = { - .y_mode = { - { CDF12(22801, 23489, 24293, 24756, 25601, 26123, - 26606, 27418, 27945, 29228, 29685, 30349) }, - { CDF12(18673, 19845, 22631, 23318, 23950, 24649, - 25527, 27364, 28152, 29701, 29984, 30852) }, - { CDF12(19770, 20979, 23396, 23939, 24241, 24654, - 25136, 27073, 27830, 29360, 29730, 30659) }, - { CDF12(20155, 21301, 22838, 23178, 23261, 23533, - 23703, 24804, 25352, 26575, 27016, 28049) }, - }, .use_filter_intra = { - [BS_4x4] = { CDF1( 4621) }, - [BS_4x8] = { CDF1( 6743) }, - [BS_8x4] = { CDF1( 5893) }, - [BS_8x8] = { CDF1( 7866) }, - [BS_8x16] = { CDF1(12551) }, - [BS_16x8] = { CDF1( 9394) }, - [BS_16x16] = { CDF1(12408) }, - [BS_16x32] = { CDF1(14301) }, - [BS_32x16] = { CDF1(12756) }, - [BS_32x32] = { CDF1(22343) }, - [BS_32x64] = { CDF1(16384) }, - [BS_64x32] = { CDF1(16384) }, - [BS_64x64] = { CDF1(16384) }, - [BS_64x128] = { CDF1(16384) }, - [BS_128x64] = { CDF1(16384) }, - [BS_128x128] = { CDF1(16384) }, - [BS_4x16] = { CDF1(12770) }, - [BS_16x4] = { CDF1(10368) }, - [BS_8x32] = { CDF1(20229) }, - [BS_32x8] = { CDF1(18101) }, - [BS_16x64] = { CDF1(16384) }, - [BS_64x16] = { CDF1(16384) }, - }, .filter_intra = { - CDF4(8949, 12776, 17211, 29558), - }, .uv_mode = { - { - { CDF12(22631, 24152, 25378, 25661, 25986, 26520, - 27055, 27923, 28244, 30059, 30941, 31961) }, - { CDF12( 9513, 26881, 26973, 27046, 27118, 27664, - 27739, 27824, 28359, 29505, 29800, 31796) }, - { CDF12( 9845, 9915, 28663, 28704, 28757, 28780, - 29198, 29822, 29854, 30764, 31777, 32029) }, - { CDF12(13639, 13897, 14171, 25331, 25606, 25727, - 25953, 27148, 28577, 30612, 31355, 32493) }, - { CDF12( 9764, 9835, 9930, 9954, 25386, 27053, - 27958, 28148, 28243, 31101, 31744, 32363) }, - { CDF12(11825, 13589, 13677, 13720, 15048, 29213, - 29301, 29458, 29711, 31161, 31441, 32550) }, - { CDF12(14175, 14399, 16608, 16821, 17718, 17775, - 28551, 30200, 30245, 31837, 32342, 32667) }, - { CDF12(12885, 13038, 14978, 15590, 15673, 15748, - 16176, 29128, 29267, 30643, 31961, 32461) }, - { CDF12(12026, 13661, 13874, 15305, 15490, 15726, - 15995, 16273, 28443, 30388, 30767, 32416) }, - { CDF12(19052, 19840, 20579, 20916, 21150, 21467, - 21885, 22719, 23174, 28861, 30379, 32175) }, - { CDF12(18627, 19649, 20974, 21219, 21492, 21816, - 22199, 23119, 23527, 27053, 31397, 32148) }, - { CDF12(17026, 19004, 19997, 20339, 20586, 21103, - 21349, 21907, 22482, 25896, 26541, 31819) }, - { CDF12(12124, 13759, 14959, 14992, 15007, 15051, - 15078, 15166, 15255, 15753, 16039, 16606) }, - }, { - { CDF13(10407, 11208, 12900, 13181, 13823, 14175, 14899, - 15656, 15986, 20086, 20995, 22455, 24212) }, - { CDF13( 4532, 19780, 20057, 20215, 20428, 21071, 21199, - 21451, 22099, 24228, 24693, 27032, 29472) }, - { CDF13( 5273, 5379, 20177, 20270, 20385, 20439, 20949, - 21695, 21774, 23138, 24256, 24703, 26679) }, - { CDF13( 6740, 7167, 7662, 14152, 14536, 14785, 15034, - 16741, 18371, 21520, 22206, 23389, 24182) }, - { CDF13( 4987, 5368, 5928, 6068, 19114, 20315, 21857, - 22253, 22411, 24911, 25380, 26027, 26376) }, - { CDF13( 5370, 6889, 7247, 7393, 9498, 21114, 21402, - 21753, 21981, 24780, 25386, 26517, 27176) }, - { CDF13( 4816, 4961, 7204, 7326, 8765, 8930, 20169, - 20682, 20803, 23188, 23763, 24455, 24940) }, - { CDF13( 6608, 6740, 8529, 9049, 9257, 9356, 9735, - 18827, 19059, 22336, 23204, 23964, 24793) }, - { CDF13( 5998, 7419, 7781, 8933, 9255, 9549, 9753, - 10417, 18898, 22494, 23139, 24764, 25989) }, - { CDF13(10660, 11298, 12550, 12957, 13322, 13624, 14040, - 15004, 15534, 20714, 21789, 23443, 24861) }, - { CDF13(10522, 11530, 12552, 12963, 13378, 13779, 14245, - 15235, 15902, 20102, 22696, 23774, 25838) }, - { CDF13(10099, 10691, 12639, 13049, 13386, 13665, 14125, - 15163, 15636, 19676, 20474, 23519, 25208) }, - { CDF13( 3144, 5087, 7382, 7504, 7593, 7690, 7801, - 8064, 8232, 9248, 9875, 10521, 29048) }, - }, - }, .angle_delta = { - { CDF6( 2180, 5032, 7567, 22776, 26989, 30217) }, - { CDF6( 2301, 5608, 8801, 23487, 26974, 30330) }, - { CDF6( 3780, 11018, 13699, 19354, 23083, 31286) }, - { CDF6( 4581, 11226, 15147, 17138, 21834, 28397) }, - { CDF6( 1737, 10927, 14509, 19588, 22745, 28823) }, - { CDF6( 2664, 10176, 12485, 17650, 21600, 30495) }, - { CDF6( 2240, 11096, 15453, 20341, 22561, 28917) }, - { CDF6( 3605, 10428, 12459, 17676, 21244, 30655) }, - }, .filter = { - { - { CDF2(31935, 32720) }, { CDF2( 5568, 32719) }, - { CDF2( 422, 2938) }, { CDF2(28244, 32608) }, - { CDF2(31206, 31953) }, { CDF2( 4862, 32121) }, - { CDF2( 770, 1152) }, { CDF2(20889, 25637) }, - }, { - { CDF2(31910, 32724) }, { CDF2( 4120, 32712) }, - { CDF2( 305, 2247) }, { CDF2(27403, 32636) }, - { CDF2(31022, 32009) }, { CDF2( 2963, 32093) }, - { CDF2( 601, 943) }, { CDF2(14969, 21398) }, - }, - }, .newmv_mode = { - { CDF1(24035) }, { CDF1(16630) }, { CDF1(15339) }, - { CDF1( 8386) }, { CDF1(12222) }, { CDF1( 4676) }, - }, .globalmv_mode = { - { CDF1( 2175) }, { CDF1( 1054) }, - }, .refmv_mode = { - { CDF1(23974) }, { CDF1(24188) }, { CDF1(17848) }, - { CDF1(28622) }, { CDF1(24312) }, { CDF1(19923) }, - }, .drl_bit = { - { CDF1(13104) }, { CDF1(24560) }, { CDF1(18945) }, - }, .comp_inter_mode = { - { CDF7( 7760, 13823, 15808, 17641, 19156, 20666, 26891) }, - { CDF7(10730, 19452, 21145, 22749, 24039, 25131, 28724) }, - { CDF7(10664, 20221, 21588, 22906, 24295, 25387, 28436) }, - { CDF7(13298, 16984, 20471, 24182, 25067, 25736, 26422) }, - { CDF7(18904, 23325, 25242, 27432, 27898, 28258, 30758) }, - { CDF7(10725, 17454, 20124, 22820, 24195, 25168, 26046) }, - { CDF7(17125, 24273, 25814, 27492, 28214, 28704, 30592) }, - { CDF7(13046, 23214, 24505, 25942, 27435, 28442, 29330) }, - }, .intra = { - { CDF1( 806) }, { CDF1(16662) }, { CDF1(20186) }, - { CDF1(26538) }, - }, .comp = { - { CDF1(26828) }, { CDF1(24035) }, { CDF1(12031) }, - { CDF1(10640) }, { CDF1( 2901) }, - }, .comp_dir = { - { CDF1( 1198) }, { CDF1( 2070) }, { CDF1( 9166) }, - { CDF1( 7499) }, { CDF1(22475) }, - }, .jnt_comp = { - { CDF1(18244) }, { CDF1(12865) }, { CDF1( 7053) }, - { CDF1(13259) }, { CDF1( 9334) }, { CDF1( 4644) }, - }, .mask_comp = { - { CDF1(26607) }, { CDF1(22891) }, { CDF1(18840) }, - { CDF1(24594) }, { CDF1(19934) }, { CDF1(22674) }, - }, .wedge_comp = { - { CDF1(23431) }, { CDF1(13171) }, { CDF1(11470) }, - { CDF1( 9770) }, { CDF1( 9100) }, { CDF1( 8233) }, - { CDF1( 6172) }, { CDF1(11820) }, { CDF1( 7701) }, - }, .wedge_idx = { - { CDF15( 2438, 4440, 6599, 8663, 11005, 12874, 15751, 18094, - 20359, 22362, 24127, 25702, 27752, 29450, 31171) }, - { CDF15( 806, 3266, 6005, 6738, 7218, 7367, 7771, 14588, - 16323, 17367, 18452, 19422, 22839, 26127, 29629) }, - { CDF15( 2779, 3738, 4683, 7213, 7775, 8017, 8655, 14357, - 17939, 21332, 24520, 27470, 29456, 30529, 31656) }, - { CDF15( 1684, 3625, 5675, 7108, 9302, 11274, 14429, 17144, - 19163, 20961, 22884, 24471, 26719, 28714, 30877) }, - { CDF15( 1142, 3491, 6277, 7314, 8089, 8355, 9023, 13624, - 15369, 16730, 18114, 19313, 22521, 26012, 29550) }, - { CDF15( 2742, 4195, 5727, 8035, 8980, 9336, 10146, 14124, - 17270, 20533, 23434, 25972, 27944, 29570, 31416) }, - { CDF15( 1727, 3948, 6101, 7796, 9841, 12344, 15766, 18944, - 20638, 22038, 23963, 25311, 26988, 28766, 31012) }, - { CDF15( 154, 987, 1925, 2051, 2088, 2111, 2151, 23033, - 23703, 24284, 24985, 25684, 27259, 28883, 30911) }, - { CDF15( 1135, 1322, 1493, 2635, 2696, 2737, 2770, 21016, - 22935, 25057, 27251, 29173, 30089, 30960, 31933) }, - }, .interintra = { - { CDF1(16384) }, { CDF1(26887) }, { CDF1(27597) }, - { CDF1(30237) }, - }, .interintra_mode = { - { CDF3(8192, 16384, 24576) }, - { CDF3(1875, 11082, 27332) }, - { CDF3(2473, 9996, 26388) }, - { CDF3(4238, 11537, 25926) }, - }, .interintra_wedge = { - { CDF1(20036) }, { CDF1(24957) }, { CDF1(26704) }, - { CDF1(27530) }, { CDF1(29564) }, { CDF1(29444) }, - { CDF1(26872) }, - }, .ref = { - { { CDF1( 4897) }, { CDF1(16973) }, { CDF1(29744) } }, - { { CDF1( 1555) }, { CDF1(16751) }, { CDF1(30279) } }, - { { CDF1( 4236) }, { CDF1(19647) }, { CDF1(31194) } }, - { { CDF1( 8650) }, { CDF1(24773) }, { CDF1(31895) } }, - { { CDF1( 904) }, { CDF1(11014) }, { CDF1(26875) } }, - { { CDF1( 1444) }, { CDF1(15087) }, { CDF1(30304) } }, - }, .comp_fwd_ref = { - { { CDF1( 4946) }, { CDF1(19891) }, { CDF1(30731) } }, - { { CDF1( 9468) }, { CDF1(22441) }, { CDF1(31059) } }, - { { CDF1( 1503) }, { CDF1(15160) }, { CDF1(27544) } }, - }, .comp_bwd_ref = { - { { CDF1( 2235) }, { CDF1(17182) }, { CDF1(30606) } }, - { { CDF1( 1423) }, { CDF1(15175) }, { CDF1(30489) } }, - }, .comp_uni_ref = { - { { CDF1( 5284) }, { CDF1(23152) }, { CDF1(31774) } }, - { { CDF1( 3865) }, { CDF1(14173) }, { CDF1(25120) } }, - { { CDF1( 3128) }, { CDF1(15270) }, { CDF1(26710) } }, - }, .txsz = { - { - { CDF1(19968) }, { CDF1(19968) }, { CDF1(24320) }, - }, { - { CDF2(12272, 30172) }, { CDF2(12272, 30172) }, - { CDF2(18677, 30848) }, - }, { - { CDF2(12986, 15180) }, { CDF2(12986, 15180) }, - { CDF2(24302, 25602) }, - }, { - { CDF2( 5782, 11475) }, { CDF2( 5782, 11475) }, - { CDF2(16803, 22759) }, - }, - }, .txpart = { - { { CDF1(28581) }, { CDF1(23846) }, { CDF1(20847) } }, - { { CDF1(24315) }, { CDF1(18196) }, { CDF1(12133) } }, - { { CDF1(18791) }, { CDF1(10887) }, { CDF1(11005) } }, - { { CDF1(27179) }, { CDF1(20004) }, { CDF1(11281) } }, - { { CDF1(26549) }, { CDF1(19308) }, { CDF1(14224) } }, - { { CDF1(28015) }, { CDF1(21546) }, { CDF1(14400) } }, - { { CDF1(28165) }, { CDF1(22401) }, { CDF1(16088) } }, - }, .txtp_inter1 = { - { CDF15( 4458, 5560, 7695, 9709, 13330, 14789, 17537, 20266, - 21504, 22848, 23934, 25474, 27727, 28915, 30631) }, - { CDF15( 1645, 2573, 4778, 5711, 7807, 8622, 10522, 15357, - 17674, 20408, 22517, 25010, 27116, 28856, 30749) }, - }, .txtp_inter2 = { - CDF11( 770, 2421, 5225, 12907, 15819, 18927, - 21561, 24089, 26595, 28526, 30529) - }, .txtp_inter3 = { - { CDF1(16384) }, { CDF1( 4167) }, { CDF1( 1998) }, { CDF1( 748) }, - }, .txtp_intra1 = { - { - { CDF6( 1535, 8035, 9461, 12751, 23467, 27825) }, - { CDF6( 564, 3335, 9709, 10870, 18143, 28094) }, - { CDF6( 672, 3247, 3676, 11982, 19415, 23127) }, - { CDF6( 5279, 13885, 15487, 18044, 23527, 30252) }, - { CDF6( 4423, 6074, 7985, 10416, 25693, 29298) }, - { CDF6( 1486, 4241, 9460, 10662, 16456, 27694) }, - { CDF6( 439, 2838, 3522, 6737, 18058, 23754) }, - { CDF6( 1190, 4233, 4855, 11670, 20281, 24377) }, - { CDF6( 1045, 4312, 8647, 10159, 18644, 29335) }, - { CDF6( 202, 3734, 4747, 7298, 17127, 24016) }, - { CDF6( 447, 4312, 6819, 8884, 16010, 23858) }, - { CDF6( 277, 4369, 5255, 8905, 16465, 22271) }, - { CDF6( 3409, 5436, 10599, 15599, 19687, 24040) }, - }, { - { CDF6( 1870, 13742, 14530, 16498, 23770, 27698) }, - { CDF6( 326, 8796, 14632, 15079, 19272, 27486) }, - { CDF6( 484, 7576, 7712, 14443, 19159, 22591) }, - { CDF6( 1126, 15340, 15895, 17023, 20896, 30279) }, - { CDF6( 655, 4854, 5249, 5913, 22099, 27138) }, - { CDF6( 1299, 6458, 8885, 9290, 14851, 25497) }, - { CDF6( 311, 5295, 5552, 6885, 16107, 22672) }, - { CDF6( 883, 8059, 8270, 11258, 17289, 21549) }, - { CDF6( 741, 7580, 9318, 10345, 16688, 29046) }, - { CDF6( 110, 7406, 7915, 9195, 16041, 23329) }, - { CDF6( 363, 7974, 9357, 10673, 15629, 24474) }, - { CDF6( 153, 7647, 8112, 9936, 15307, 19996) }, - { CDF6( 3511, 6332, 11165, 15335, 19323, 23594) }, - }, - }, .txtp_intra2 = { - { - { CDF4( 6554, 13107, 19661, 26214) }, - { CDF4( 6554, 13107, 19661, 26214) }, - { CDF4( 6554, 13107, 19661, 26214) }, - { CDF4( 6554, 13107, 19661, 26214) }, - { CDF4( 6554, 13107, 19661, 26214) }, - { CDF4( 6554, 13107, 19661, 26214) }, - { CDF4( 6554, 13107, 19661, 26214) }, - { CDF4( 6554, 13107, 19661, 26214) }, - { CDF4( 6554, 13107, 19661, 26214) }, - { CDF4( 6554, 13107, 19661, 26214) }, - { CDF4( 6554, 13107, 19661, 26214) }, - { CDF4( 6554, 13107, 19661, 26214) }, - { CDF4( 6554, 13107, 19661, 26214) }, - }, { - { CDF4( 6554, 13107, 19661, 26214) }, - { CDF4( 6554, 13107, 19661, 26214) }, - { CDF4( 6554, 13107, 19661, 26214) }, - { CDF4( 6554, 13107, 19661, 26214) }, - { CDF4( 6554, 13107, 19661, 26214) }, - { CDF4( 6554, 13107, 19661, 26214) }, - { CDF4( 6554, 13107, 19661, 26214) }, - { CDF4( 6554, 13107, 19661, 26214) }, - { CDF4( 6554, 13107, 19661, 26214) }, - { CDF4( 6554, 13107, 19661, 26214) }, - { CDF4( 6554, 13107, 19661, 26214) }, - { CDF4( 6554, 13107, 19661, 26214) }, - { CDF4( 6554, 13107, 19661, 26214) }, - }, { - { CDF4( 1127, 12814, 22772, 27483) }, - { CDF4( 145, 6761, 11980, 26667) }, - { CDF4( 362, 5887, 11678, 16725) }, - { CDF4( 385, 15213, 18587, 30693) }, - { CDF4( 25, 2914, 23134, 27903) }, - { CDF4( 60, 4470, 11749, 23991) }, - { CDF4( 37, 3332, 14511, 21448) }, - { CDF4( 157, 6320, 13036, 17439) }, - { CDF4( 119, 6719, 12906, 29396) }, - { CDF4( 47, 5537, 12576, 21499) }, - { CDF4( 269, 6076, 11258, 23115) }, - { CDF4( 83, 5615, 12001, 17228) }, - { CDF4( 1968, 5556, 12023, 18547) }, - }, - }, .skip = { - { CDF1(31671) }, { CDF1(16515) }, { CDF1( 4576) }, - }, .skip_mode = { - { CDF1(32621) }, { CDF1(20708) }, { CDF1( 8127) }, - }, .partition = { - { - // 128x128 -> 64x64 - { CDF7(27899, 28219, 28529, 32484, 32539, 32619, 32639) }, - { CDF7( 6607, 6990, 8268, 32060, 32219, 32338, 32371) }, - { CDF7( 5429, 6676, 7122, 32027, 32227, 32531, 32582) }, - { CDF7( 711, 966, 1172, 32448, 32538, 32617, 32664) }, - }, { - // 64x64 -> 32x32 - { CDF9(20137, 21547, 23078, 29566, 29837, - 30261, 30524, 30892, 31724) }, - { CDF9( 6732, 7490, 9497, 27944, 28250, - 28515, 28969, 29630, 30104) }, - { CDF9( 5945, 7663, 8348, 28683, 29117, - 29749, 30064, 30298, 32238) }, - { CDF9( 870, 1212, 1487, 31198, 31394, - 31574, 31743, 31881, 32332) }, - }, { - // 32x32 -> 16x16 - { CDF9(18462, 20920, 23124, 27647, 28227, - 29049, 29519, 30178, 31544) }, - { CDF9( 7689, 9060, 12056, 24992, 25660, - 26182, 26951, 28041, 29052) }, - { CDF9( 6015, 9009, 10062, 24544, 25409, - 26545, 27071, 27526, 32047) }, - { CDF9( 1394, 2208, 2796, 28614, 29061, - 29466, 29840, 30185, 31899) }, - }, { - // 16x16 -> 8x8 - { CDF9(15597, 20929, 24571, 26706, 27664, - 28821, 29601, 30571, 31902) }, - { CDF9( 7925, 11043, 16785, 22470, 23971, - 25043, 26651, 28701, 29834) }, - { CDF9( 5414, 13269, 15111, 20488, 22360, - 24500, 25537, 26336, 32117) }, - { CDF9( 2662, 6362, 8614, 20860, 23053, - 24778, 26436, 27829, 31171) }, - }, { - // 8x8 -> 4x4 only supports the four legacy partition types - { CDF3(19132, 25510, 30392) }, - { CDF3(13928, 19855, 28540) }, - { CDF3(12522, 23679, 28629) }, - { CDF3( 9896, 18783, 25853) }, - }, - }, .seg_pred = { - { CDF1(16384) }, { CDF1(16384) }, { CDF1(16384) }, - }, .seg_id = { - { CDF7( 5622, 7893, 16093, 18233, 27809, 28373, 32533) }, - { CDF7(14274, 18230, 22557, 24935, 29980, 30851, 32344) }, - { CDF7(27527, 28487, 28723, 28890, 32397, 32647, 32679) }, - }, .cfl_sign = { - CDF7( 1418, 2123, 13340, 18405, 26972, 28343, 32294) - }, .cfl_alpha = { - { CDF15( 7637, 20719, 31401, 32481, 32657, 32688, 32692, 32696, - 32700, 32704, 32708, 32712, 32716, 32720, 32724) }, - { CDF15(14365, 23603, 28135, 31168, 32167, 32395, 32487, 32573, - 32620, 32647, 32668, 32672, 32676, 32680, 32684) }, - { CDF15(11532, 22380, 28445, 31360, 32349, 32523, 32584, 32649, - 32673, 32677, 32681, 32685, 32689, 32693, 32697) }, - { CDF15(26990, 31402, 32282, 32571, 32692, 32696, 32700, 32704, - 32708, 32712, 32716, 32720, 32724, 32728, 32732) }, - { CDF15(17248, 26058, 28904, 30608, 31305, 31877, 32126, 32321, - 32394, 32464, 32516, 32560, 32576, 32593, 32622) }, - { CDF15(14738, 21678, 25779, 27901, 29024, 30302, 30980, 31843, - 32144, 32413, 32520, 32594, 32622, 32656, 32660) }, - }, .restore_wiener = { - CDF1(11570) - }, .restore_sgrproj = { - CDF1(16855) - }, .restore_switchable = { - CDF2( 9413, 22581) - }, .delta_q = { - CDF3(28160, 32120, 32677) - }, .delta_lf = { - { CDF3(28160, 32120, 32677) }, - { CDF3(28160, 32120, 32677) }, - { CDF3(28160, 32120, 32677) }, - { CDF3(28160, 32120, 32677) }, - { CDF3(28160, 32120, 32677) }, - }, .motion_mode = { - [BS_8x8] = { CDF2( 7651, 24760) }, - [BS_8x16] = { CDF2( 4738, 24765) }, - [BS_8x32] = { CDF2(28799, 31390) }, - [BS_16x8] = { CDF2( 5391, 25528) }, - [BS_16x16] = { CDF2(19419, 26810) }, - [BS_16x32] = { CDF2( 5123, 23606) }, - [BS_16x64] = { CDF2(28973, 31594) }, - [BS_32x8] = { CDF2(26431, 30774) }, - [BS_32x16] = { CDF2(11606, 24308) }, - [BS_32x32] = { CDF2(26260, 29116) }, - [BS_32x64] = { CDF2(20360, 28062) }, - [BS_64x16] = { CDF2(29742, 31203) }, - [BS_64x32] = { CDF2(21679, 26830) }, - [BS_64x64] = { CDF2(29516, 30701) }, - [BS_64x128] = { CDF2(28898, 30397) }, - [BS_128x64] = { CDF2(30878, 31335) }, - [BS_128x128] = { CDF2(32507, 32558) }, - }, .obmc = { - [BS_8x8] = { CDF1(10437) }, - [BS_8x16] = { CDF1( 9371) }, - [BS_8x32] = { CDF1(23664) }, - [BS_16x8] = { CDF1( 9301) }, - [BS_16x16] = { CDF1(17432) }, - [BS_16x32] = { CDF1(14423) }, - [BS_16x64] = { CDF1(24008) }, - [BS_32x8] = { CDF1(20901) }, - [BS_32x16] = { CDF1(15142) }, - [BS_32x32] = { CDF1(25817) }, - [BS_32x64] = { CDF1(22823) }, - [BS_64x16] = { CDF1(26879) }, - [BS_64x32] = { CDF1(22083) }, - [BS_64x64] = { CDF1(30128) }, - [BS_64x128] = { CDF1(31014) }, - [BS_128x64] = { CDF1(31560) }, - [BS_128x128] = { CDF1(32638) }, - }, .pal_y = { - { { CDF1(31676) }, { CDF1( 3419) }, { CDF1( 1261) } }, - { { CDF1(31912) }, { CDF1( 2859) }, { CDF1( 980) } }, - { { CDF1(31823) }, { CDF1( 3400) }, { CDF1( 781) } }, - { { CDF1(32030) }, { CDF1( 3561) }, { CDF1( 904) } }, - { { CDF1(32309) }, { CDF1( 7337) }, { CDF1( 1462) } }, - { { CDF1(32265) }, { CDF1( 4015) }, { CDF1( 1521) } }, - { { CDF1(32450) }, { CDF1( 7946) }, { CDF1( 129) } }, - }, .pal_sz = { - { - { CDF6( 7952, 13000, 18149, 21478, 25527, 29241) }, - { CDF6( 7139, 11421, 16195, 19544, 23666, 28073) }, - { CDF6( 7788, 12741, 17325, 20500, 24315, 28530) }, - { CDF6( 8271, 14064, 18246, 21564, 25071, 28533) }, - { CDF6(12725, 19180, 21863, 24839, 27535, 30120) }, - { CDF6( 9711, 14888, 16923, 21052, 25661, 27875) }, - { CDF6(14940, 20797, 21678, 24186, 27033, 28999) }, - }, { - { CDF6( 8713, 19979, 27128, 29609, 31331, 32272) }, - { CDF6( 5839, 15573, 23581, 26947, 29848, 31700) }, - { CDF6( 4426, 11260, 17999, 21483, 25863, 29430) }, - { CDF6( 3228, 9464, 14993, 18089, 22523, 27420) }, - { CDF6( 3768, 8886, 13091, 17852, 22495, 27207) }, - { CDF6( 2464, 8451, 12861, 21632, 25525, 28555) }, - { CDF6( 1269, 5435, 10433, 18963, 21700, 25865) }, - }, - }, .pal_uv = { - { CDF1(32461) }, { CDF1(21488) }, - }, .color_map = { - { /* y */ +typedef struct CdfDefaultContext { + CdfModeContext m; + struct { + CdfMvComponent comp; + ALIGN(uint16_t joint[N_MV_JOINTS], 8); + } mv; + ALIGN(uint16_t kfym[5][5][N_INTRA_PRED_MODES + 3], 32); +} CdfDefaultContext; + +static const CdfDefaultContext default_cdf = { + .m = { + .y_mode = { + { CDF12(22801, 23489, 24293, 24756, 25601, 26123, + 26606, 27418, 27945, 29228, 29685, 30349) }, + { CDF12(18673, 19845, 22631, 23318, 23950, 24649, + 25527, 27364, 28152, 29701, 29984, 30852) }, + { CDF12(19770, 20979, 23396, 23939, 24241, 24654, + 25136, 27073, 27830, 29360, 29730, 30659) }, + { CDF12(20155, 21301, 22838, 23178, 23261, 23533, + 23703, 24804, 25352, 26575, 27016, 28049) }, + }, .use_filter_intra = { + [BS_4x4] = { CDF1( 4621) }, + [BS_4x8] = { CDF1( 6743) }, + [BS_8x4] = { CDF1( 5893) }, + [BS_8x8] = { CDF1( 7866) }, + [BS_8x16] = { CDF1(12551) }, + [BS_16x8] = { CDF1( 9394) }, + [BS_16x16] = { CDF1(12408) }, + [BS_16x32] = { CDF1(14301) }, + [BS_32x16] = { CDF1(12756) }, + [BS_32x32] = { CDF1(22343) }, + [BS_32x64] = { CDF1(16384) }, + [BS_64x32] = { CDF1(16384) }, + [BS_64x64] = { CDF1(16384) }, + [BS_64x128] = { CDF1(16384) }, + [BS_128x64] = { CDF1(16384) }, + [BS_128x128] = { CDF1(16384) }, + [BS_4x16] = { CDF1(12770) }, + [BS_16x4] = { CDF1(10368) }, + [BS_8x32] = { CDF1(20229) }, + [BS_32x8] = { CDF1(18101) }, + [BS_16x64] = { CDF1(16384) }, + [BS_64x16] = { CDF1(16384) }, + }, .filter_intra = { + CDF4(8949, 12776, 17211, 29558), + }, .uv_mode = { { - { CDF1(28710) }, { CDF1(16384) }, { CDF1(10553) }, - { CDF1(27036) }, { CDF1(31603) }, + { CDF12(22631, 24152, 25378, 25661, 25986, 26520, + 27055, 27923, 28244, 30059, 30941, 31961) }, + { CDF12( 9513, 26881, 26973, 27046, 27118, 27664, + 27739, 27824, 28359, 29505, 29800, 31796) }, + { CDF12( 9845, 9915, 28663, 28704, 28757, 28780, + 29198, 29822, 29854, 30764, 31777, 32029) }, + { CDF12(13639, 13897, 14171, 25331, 25606, 25727, + 25953, 27148, 28577, 30612, 31355, 32493) }, + { CDF12( 9764, 9835, 9930, 9954, 25386, 27053, + 27958, 28148, 28243, 31101, 31744, 32363) }, + { CDF12(11825, 13589, 13677, 13720, 15048, 29213, + 29301, 29458, 29711, 31161, 31441, 32550) }, + { CDF12(14175, 14399, 16608, 16821, 17718, 17775, + 28551, 30200, 30245, 31837, 32342, 32667) }, + { CDF12(12885, 13038, 14978, 15590, 15673, 15748, + 16176, 29128, 29267, 30643, 31961, 32461) }, + { CDF12(12026, 13661, 13874, 15305, 15490, 15726, + 15995, 16273, 28443, 30388, 30767, 32416) }, + { CDF12(19052, 19840, 20579, 20916, 21150, 21467, + 21885, 22719, 23174, 28861, 30379, 32175) }, + { CDF12(18627, 19649, 20974, 21219, 21492, 21816, + 22199, 23119, 23527, 27053, 31397, 32148) }, + { CDF12(17026, 19004, 19997, 20339, 20586, 21103, + 21349, 21907, 22482, 25896, 26541, 31819) }, + { CDF12(12124, 13759, 14959, 14992, 15007, 15051, + 15078, 15166, 15255, 15753, 16039, 16606) }, }, { - { CDF2(27877, 30490) }, { CDF2(11532, 25697) }, - { CDF2( 6544, 30234) }, { CDF2(23018, 28072) }, - { CDF2(31915, 32385) }, + { CDF13(10407, 11208, 12900, 13181, 13823, 14175, 14899, + 15656, 15986, 20086, 20995, 22455, 24212) }, + { CDF13( 4532, 19780, 20057, 20215, 20428, 21071, 21199, + 21451, 22099, 24228, 24693, 27032, 29472) }, + { CDF13( 5273, 5379, 20177, 20270, 20385, 20439, 20949, + 21695, 21774, 23138, 24256, 24703, 26679) }, + { CDF13( 6740, 7167, 7662, 14152, 14536, 14785, 15034, + 16741, 18371, 21520, 22206, 23389, 24182) }, + { CDF13( 4987, 5368, 5928, 6068, 19114, 20315, 21857, + 22253, 22411, 24911, 25380, 26027, 26376) }, + { CDF13( 5370, 6889, 7247, 7393, 9498, 21114, 21402, + 21753, 21981, 24780, 25386, 26517, 27176) }, + { CDF13( 4816, 4961, 7204, 7326, 8765, 8930, 20169, + 20682, 20803, 23188, 23763, 24455, 24940) }, + { CDF13( 6608, 6740, 8529, 9049, 9257, 9356, 9735, + 18827, 19059, 22336, 23204, 23964, 24793) }, + { CDF13( 5998, 7419, 7781, 8933, 9255, 9549, 9753, + 10417, 18898, 22494, 23139, 24764, 25989) }, + { CDF13(10660, 11298, 12550, 12957, 13322, 13624, 14040, + 15004, 15534, 20714, 21789, 23443, 24861) }, + { CDF13(10522, 11530, 12552, 12963, 13378, 13779, 14245, + 15235, 15902, 20102, 22696, 23774, 25838) }, + { CDF13(10099, 10691, 12639, 13049, 13386, 13665, 14125, + 15163, 15636, 19676, 20474, 23519, 25208) }, + { CDF13( 3144, 5087, 7382, 7504, 7593, 7690, 7801, + 8064, 8232, 9248, 9875, 10521, 29048) }, + }, + }, .angle_delta = { + { CDF6( 2180, 5032, 7567, 22776, 26989, 30217) }, + { CDF6( 2301, 5608, 8801, 23487, 26974, 30330) }, + { CDF6( 3780, 11018, 13699, 19354, 23083, 31286) }, + { CDF6( 4581, 11226, 15147, 17138, 21834, 28397) }, + { CDF6( 1737, 10927, 14509, 19588, 22745, 28823) }, + { CDF6( 2664, 10176, 12485, 17650, 21600, 30495) }, + { CDF6( 2240, 11096, 15453, 20341, 22561, 28917) }, + { CDF6( 3605, 10428, 12459, 17676, 21244, 30655) }, + }, .filter = { + { + { CDF2(31935, 32720) }, { CDF2( 5568, 32719) }, + { CDF2( 422, 2938) }, { CDF2(28244, 32608) }, + { CDF2(31206, 31953) }, { CDF2( 4862, 32121) }, + { CDF2( 770, 1152) }, { CDF2(20889, 25637) }, }, { - { CDF3(25572, 28046, 30045) }, - { CDF3( 9478, 21590, 27256) }, - { CDF3( 7248, 26837, 29824) }, - { CDF3(19167, 24486, 28349) }, - { CDF3(31400, 31825, 32250) }, + { CDF2(31910, 32724) }, { CDF2( 4120, 32712) }, + { CDF2( 305, 2247) }, { CDF2(27403, 32636) }, + { CDF2(31022, 32009) }, { CDF2( 2963, 32093) }, + { CDF2( 601, 943) }, { CDF2(14969, 21398) }, + }, + }, .newmv_mode = { + { CDF1(24035) }, { CDF1(16630) }, { CDF1(15339) }, + { CDF1( 8386) }, { CDF1(12222) }, { CDF1( 4676) }, + }, .globalmv_mode = { + { CDF1( 2175) }, { CDF1( 1054) }, + }, .refmv_mode = { + { CDF1(23974) }, { CDF1(24188) }, { CDF1(17848) }, + { CDF1(28622) }, { CDF1(24312) }, { CDF1(19923) }, + }, .drl_bit = { + { CDF1(13104) }, { CDF1(24560) }, { CDF1(18945) }, + }, .comp_inter_mode = { + { CDF7( 7760, 13823, 15808, 17641, 19156, 20666, 26891) }, + { CDF7(10730, 19452, 21145, 22749, 24039, 25131, 28724) }, + { CDF7(10664, 20221, 21588, 22906, 24295, 25387, 28436) }, + { CDF7(13298, 16984, 20471, 24182, 25067, 25736, 26422) }, + { CDF7(18904, 23325, 25242, 27432, 27898, 28258, 30758) }, + { CDF7(10725, 17454, 20124, 22820, 24195, 25168, 26046) }, + { CDF7(17125, 24273, 25814, 27492, 28214, 28704, 30592) }, + { CDF7(13046, 23214, 24505, 25942, 27435, 28442, 29330) }, + }, .intra = { + { CDF1( 806) }, { CDF1(16662) }, { CDF1(20186) }, + { CDF1(26538) }, + }, .comp = { + { CDF1(26828) }, { CDF1(24035) }, { CDF1(12031) }, + { CDF1(10640) }, { CDF1( 2901) }, + }, .comp_dir = { + { CDF1( 1198) }, { CDF1( 2070) }, { CDF1( 9166) }, + { CDF1( 7499) }, { CDF1(22475) }, + }, .jnt_comp = { + { CDF1(18244) }, { CDF1(12865) }, { CDF1( 7053) }, + { CDF1(13259) }, { CDF1( 9334) }, { CDF1( 4644) }, + }, .mask_comp = { + { CDF1(26607) }, { CDF1(22891) }, { CDF1(18840) }, + { CDF1(24594) }, { CDF1(19934) }, { CDF1(22674) }, + }, .wedge_comp = { + { CDF1(23431) }, { CDF1(13171) }, { CDF1(11470) }, + { CDF1( 9770) }, { CDF1( 9100) }, { CDF1( 8233) }, + { CDF1( 6172) }, { CDF1(11820) }, { CDF1( 7701) }, + }, .wedge_idx = { + { CDF15( 2438, 4440, 6599, 8663, 11005, 12874, 15751, 18094, + 20359, 22362, 24127, 25702, 27752, 29450, 31171) }, + { CDF15( 806, 3266, 6005, 6738, 7218, 7367, 7771, 14588, + 16323, 17367, 18452, 19422, 22839, 26127, 29629) }, + { CDF15( 2779, 3738, 4683, 7213, 7775, 8017, 8655, 14357, + 17939, 21332, 24520, 27470, 29456, 30529, 31656) }, + { CDF15( 1684, 3625, 5675, 7108, 9302, 11274, 14429, 17144, + 19163, 20961, 22884, 24471, 26719, 28714, 30877) }, + { CDF15( 1142, 3491, 6277, 7314, 8089, 8355, 9023, 13624, + 15369, 16730, 18114, 19313, 22521, 26012, 29550) }, + { CDF15( 2742, 4195, 5727, 8035, 8980, 9336, 10146, 14124, + 17270, 20533, 23434, 25972, 27944, 29570, 31416) }, + { CDF15( 1727, 3948, 6101, 7796, 9841, 12344, 15766, 18944, + 20638, 22038, 23963, 25311, 26988, 28766, 31012) }, + { CDF15( 154, 987, 1925, 2051, 2088, 2111, 2151, 23033, + 23703, 24284, 24985, 25684, 27259, 28883, 30911) }, + { CDF15( 1135, 1322, 1493, 2635, 2696, 2737, 2770, 21016, + 22935, 25057, 27251, 29173, 30089, 30960, 31933) }, + }, .interintra = { + { CDF1(16384) }, { CDF1(26887) }, { CDF1(27597) }, + { CDF1(30237) }, + }, .interintra_mode = { + { CDF3(8192, 16384, 24576) }, + { CDF3(1875, 11082, 27332) }, + { CDF3(2473, 9996, 26388) }, + { CDF3(4238, 11537, 25926) }, + }, .interintra_wedge = { + { CDF1(20036) }, { CDF1(24957) }, { CDF1(26704) }, + { CDF1(27530) }, { CDF1(29564) }, { CDF1(29444) }, + { CDF1(26872) }, + }, .ref = { + { { CDF1( 4897) }, { CDF1(16973) }, { CDF1(29744) } }, + { { CDF1( 1555) }, { CDF1(16751) }, { CDF1(30279) } }, + { { CDF1( 4236) }, { CDF1(19647) }, { CDF1(31194) } }, + { { CDF1( 8650) }, { CDF1(24773) }, { CDF1(31895) } }, + { { CDF1( 904) }, { CDF1(11014) }, { CDF1(26875) } }, + { { CDF1( 1444) }, { CDF1(15087) }, { CDF1(30304) } }, + }, .comp_fwd_ref = { + { { CDF1( 4946) }, { CDF1(19891) }, { CDF1(30731) } }, + { { CDF1( 9468) }, { CDF1(22441) }, { CDF1(31059) } }, + { { CDF1( 1503) }, { CDF1(15160) }, { CDF1(27544) } }, + }, .comp_bwd_ref = { + { { CDF1( 2235) }, { CDF1(17182) }, { CDF1(30606) } }, + { { CDF1( 1423) }, { CDF1(15175) }, { CDF1(30489) } }, + }, .comp_uni_ref = { + { { CDF1( 5284) }, { CDF1(23152) }, { CDF1(31774) } }, + { { CDF1( 3865) }, { CDF1(14173) }, { CDF1(25120) } }, + { { CDF1( 3128) }, { CDF1(15270) }, { CDF1(26710) } }, + }, .txsz = { + { + { CDF1(19968) }, { CDF1(19968) }, { CDF1(24320) }, }, { - { CDF4(24779, 26955, 28576, 30282) }, - { CDF4( 8669, 20364, 24073, 28093) }, - { CDF4( 4255, 27565, 29377, 31067) }, - { CDF4(19864, 23674, 26716, 29530) }, - { CDF4(31646, 31893, 32147, 32426) }, + { CDF2(12272, 30172) }, { CDF2(12272, 30172) }, + { CDF2(18677, 30848) }, }, { - { CDF5(23132, 25407, 26970, 28435, 30073) }, - { CDF5( 7443, 17242, 20717, 24762, 27982) }, - { CDF5( 6300, 24862, 26944, 28784, 30671) }, - { CDF5(18916, 22895, 25267, 27435, 29652) }, - { CDF5(31270, 31550, 31808, 32059, 32353) }, + { CDF2(12986, 15180) }, { CDF2(12986, 15180) }, + { CDF2(24302, 25602) }, }, { - { CDF6(23105, 25199, 26464, 27684, 28931, 30318) }, - { CDF6( 6950, 15447, 18952, 22681, 25567, 28563) }, - { CDF6( 7560, 23474, 25490, 27203, 28921, 30708) }, - { CDF6(18544, 22373, 24457, 26195, 28119, 30045) }, - { CDF6(31198, 31451, 31670, 31882, 32123, 32391) }, + { CDF2( 5782, 11475) }, { CDF2( 5782, 11475) }, + { CDF2(16803, 22759) }, + }, + }, .txpart = { + { { CDF1(28581) }, { CDF1(23846) }, { CDF1(20847) } }, + { { CDF1(24315) }, { CDF1(18196) }, { CDF1(12133) } }, + { { CDF1(18791) }, { CDF1(10887) }, { CDF1(11005) } }, + { { CDF1(27179) }, { CDF1(20004) }, { CDF1(11281) } }, + { { CDF1(26549) }, { CDF1(19308) }, { CDF1(14224) } }, + { { CDF1(28015) }, { CDF1(21546) }, { CDF1(14400) } }, + { { CDF1(28165) }, { CDF1(22401) }, { CDF1(16088) } }, + }, .txtp_inter1 = { + { CDF15( 4458, 5560, 7695, 9709, 13330, 14789, 17537, 20266, + 21504, 22848, 23934, 25474, 27727, 28915, 30631) }, + { CDF15( 1645, 2573, 4778, 5711, 7807, 8622, 10522, 15357, + 17674, 20408, 22517, 25010, 27116, 28856, 30749) }, + }, .txtp_inter2 = { + CDF11( 770, 2421, 5225, 12907, 15819, 18927, + 21561, 24089, 26595, 28526, 30529) + }, .txtp_inter3 = { + { CDF1(16384) }, { CDF1( 4167) }, { CDF1( 1998) }, { CDF1( 748) }, + }, .txtp_intra1 = { + { + { CDF6( 1535, 8035, 9461, 12751, 23467, 27825) }, + { CDF6( 564, 3335, 9709, 10870, 18143, 28094) }, + { CDF6( 672, 3247, 3676, 11982, 19415, 23127) }, + { CDF6( 5279, 13885, 15487, 18044, 23527, 30252) }, + { CDF6( 4423, 6074, 7985, 10416, 25693, 29298) }, + { CDF6( 1486, 4241, 9460, 10662, 16456, 27694) }, + { CDF6( 439, 2838, 3522, 6737, 18058, 23754) }, + { CDF6( 1190, 4233, 4855, 11670, 20281, 24377) }, + { CDF6( 1045, 4312, 8647, 10159, 18644, 29335) }, + { CDF6( 202, 3734, 4747, 7298, 17127, 24016) }, + { CDF6( 447, 4312, 6819, 8884, 16010, 23858) }, + { CDF6( 277, 4369, 5255, 8905, 16465, 22271) }, + { CDF6( 3409, 5436, 10599, 15599, 19687, 24040) }, }, { - { CDF7(21689, 23883, 25163, 26352, 27506, 28827, 30195) }, - { CDF7( 6892, 15385, 17840, 21606, 24287, 26753, 29204) }, - { CDF7( 5651, 23182, 25042, 26518, 27982, 29392, 30900) }, - { CDF7(19349, 22578, 24418, 25994, 27524, 29031, 30448) }, - { CDF7(31028, 31270, 31504, 31705, 31927, 32153, 32392) }, + { CDF6( 1870, 13742, 14530, 16498, 23770, 27698) }, + { CDF6( 326, 8796, 14632, 15079, 19272, 27486) }, + { CDF6( 484, 7576, 7712, 14443, 19159, 22591) }, + { CDF6( 1126, 15340, 15895, 17023, 20896, 30279) }, + { CDF6( 655, 4854, 5249, 5913, 22099, 27138) }, + { CDF6( 1299, 6458, 8885, 9290, 14851, 25497) }, + { CDF6( 311, 5295, 5552, 6885, 16107, 22672) }, + { CDF6( 883, 8059, 8270, 11258, 17289, 21549) }, + { CDF6( 741, 7580, 9318, 10345, 16688, 29046) }, + { CDF6( 110, 7406, 7915, 9195, 16041, 23329) }, + { CDF6( 363, 7974, 9357, 10673, 15629, 24474) }, + { CDF6( 153, 7647, 8112, 9936, 15307, 19996) }, + { CDF6( 3511, 6332, 11165, 15335, 19323, 23594) }, }, - }, { /* uv */ + }, .txtp_intra2 = { { - { CDF1(29089) }, { CDF1(16384) }, { CDF1( 8713) }, - { CDF1(29257) }, { CDF1(31610) }, + { CDF4( 6554, 13107, 19661, 26214) }, + { CDF4( 6554, 13107, 19661, 26214) }, + { CDF4( 6554, 13107, 19661, 26214) }, + { CDF4( 6554, 13107, 19661, 26214) }, + { CDF4( 6554, 13107, 19661, 26214) }, + { CDF4( 6554, 13107, 19661, 26214) }, + { CDF4( 6554, 13107, 19661, 26214) }, + { CDF4( 6554, 13107, 19661, 26214) }, + { CDF4( 6554, 13107, 19661, 26214) }, + { CDF4( 6554, 13107, 19661, 26214) }, + { CDF4( 6554, 13107, 19661, 26214) }, + { CDF4( 6554, 13107, 19661, 26214) }, + { CDF4( 6554, 13107, 19661, 26214) }, + }, { + { CDF4( 6554, 13107, 19661, 26214) }, + { CDF4( 6554, 13107, 19661, 26214) }, + { CDF4( 6554, 13107, 19661, 26214) }, + { CDF4( 6554, 13107, 19661, 26214) }, + { CDF4( 6554, 13107, 19661, 26214) }, + { CDF4( 6554, 13107, 19661, 26214) }, + { CDF4( 6554, 13107, 19661, 26214) }, + { CDF4( 6554, 13107, 19661, 26214) }, + { CDF4( 6554, 13107, 19661, 26214) }, + { CDF4( 6554, 13107, 19661, 26214) }, + { CDF4( 6554, 13107, 19661, 26214) }, + { CDF4( 6554, 13107, 19661, 26214) }, + { CDF4( 6554, 13107, 19661, 26214) }, }, { - { CDF2(25257, 29145) }, { CDF2(12287, 27293) }, - { CDF2( 7033, 27960) }, { CDF2(20145, 25405) }, - { CDF2(30608, 31639) }, + { CDF4( 1127, 12814, 22772, 27483) }, + { CDF4( 145, 6761, 11980, 26667) }, + { CDF4( 362, 5887, 11678, 16725) }, + { CDF4( 385, 15213, 18587, 30693) }, + { CDF4( 25, 2914, 23134, 27903) }, + { CDF4( 60, 4470, 11749, 23991) }, + { CDF4( 37, 3332, 14511, 21448) }, + { CDF4( 157, 6320, 13036, 17439) }, + { CDF4( 119, 6719, 12906, 29396) }, + { CDF4( 47, 5537, 12576, 21499) }, + { CDF4( 269, 6076, 11258, 23115) }, + { CDF4( 83, 5615, 12001, 17228) }, + { CDF4( 1968, 5556, 12023, 18547) }, + }, + }, .skip = { + { CDF1(31671) }, { CDF1(16515) }, { CDF1( 4576) }, + }, .skip_mode = { + { CDF1(32621) }, { CDF1(20708) }, { CDF1( 8127) }, + }, .partition = { + { + // 128x128 -> 64x64 + { CDF7(27899, 28219, 28529, 32484, 32539, 32619, 32639) }, + { CDF7( 6607, 6990, 8268, 32060, 32219, 32338, 32371) }, + { CDF7( 5429, 6676, 7122, 32027, 32227, 32531, 32582) }, + { CDF7( 711, 966, 1172, 32448, 32538, 32617, 32664) }, }, { - { CDF3(24210, 27175, 29903) }, - { CDF3( 9888, 22386, 27214) }, - { CDF3( 5901, 26053, 29293) }, - { CDF3(18318, 22152, 28333) }, - { CDF3(30459, 31136, 31926) }, + // 64x64 -> 32x32 + { CDF9(20137, 21547, 23078, 29566, 29837, + 30261, 30524, 30892, 31724) }, + { CDF9( 6732, 7490, 9497, 27944, 28250, + 28515, 28969, 29630, 30104) }, + { CDF9( 5945, 7663, 8348, 28683, 29117, + 29749, 30064, 30298, 32238) }, + { CDF9( 870, 1212, 1487, 31198, 31394, + 31574, 31743, 31881, 32332) }, }, { - { CDF4(22980, 25479, 27781, 29986) }, - { CDF4( 8413, 21408, 24859, 28874) }, - { CDF4( 2257, 29449, 30594, 31598) }, - { CDF4(19189, 21202, 25915, 28620) }, - { CDF4(31844, 32044, 32281, 32518) }, + // 32x32 -> 16x16 + { CDF9(18462, 20920, 23124, 27647, 28227, + 29049, 29519, 30178, 31544) }, + { CDF9( 7689, 9060, 12056, 24992, 25660, + 26182, 26951, 28041, 29052) }, + { CDF9( 6015, 9009, 10062, 24544, 25409, + 26545, 27071, 27526, 32047) }, + { CDF9( 1394, 2208, 2796, 28614, 29061, + 29466, 29840, 30185, 31899) }, }, { - { CDF5(22217, 24567, 26637, 28683, 30548) }, - { CDF5( 7307, 16406, 19636, 24632, 28424) }, - { CDF5( 4441, 25064, 26879, 28942, 30919) }, - { CDF5(17210, 20528, 23319, 26750, 29582) }, - { CDF5(30674, 30953, 31396, 31735, 32207) }, + // 16x16 -> 8x8 + { CDF9(15597, 20929, 24571, 26706, 27664, + 28821, 29601, 30571, 31902) }, + { CDF9( 7925, 11043, 16785, 22470, 23971, + 25043, 26651, 28701, 29834) }, + { CDF9( 5414, 13269, 15111, 20488, 22360, + 24500, 25537, 26336, 32117) }, + { CDF9( 2662, 6362, 8614, 20860, 23053, + 24778, 26436, 27829, 31171) }, }, { - { CDF6(21239, 23168, 25044, 26962, 28705, 30506) }, - { CDF6( 6545, 15012, 18004, 21817, 25503, 28701) }, - { CDF6( 3448, 26295, 27437, 28704, 30126, 31442) }, - { CDF6(15889, 18323, 21704, 24698, 26976, 29690) }, - { CDF6(30988, 31204, 31479, 31734, 31983, 32325) }, + // 8x8 -> 4x4 only supports the four legacy partition types + { CDF3(19132, 25510, 30392) }, + { CDF3(13928, 19855, 28540) }, + { CDF3(12522, 23679, 28629) }, + { CDF3( 9896, 18783, 25853) }, + }, + }, .seg_pred = { + { CDF1(16384) }, { CDF1(16384) }, { CDF1(16384) }, + }, .seg_id = { + { CDF7( 5622, 7893, 16093, 18233, 27809, 28373, 32533) }, + { CDF7(14274, 18230, 22557, 24935, 29980, 30851, 32344) }, + { CDF7(27527, 28487, 28723, 28890, 32397, 32647, 32679) }, + }, .cfl_sign = { + CDF7( 1418, 2123, 13340, 18405, 26972, 28343, 32294) + }, .cfl_alpha = { + { CDF15( 7637, 20719, 31401, 32481, 32657, 32688, 32692, 32696, + 32700, 32704, 32708, 32712, 32716, 32720, 32724) }, + { CDF15(14365, 23603, 28135, 31168, 32167, 32395, 32487, 32573, + 32620, 32647, 32668, 32672, 32676, 32680, 32684) }, + { CDF15(11532, 22380, 28445, 31360, 32349, 32523, 32584, 32649, + 32673, 32677, 32681, 32685, 32689, 32693, 32697) }, + { CDF15(26990, 31402, 32282, 32571, 32692, 32696, 32700, 32704, + 32708, 32712, 32716, 32720, 32724, 32728, 32732) }, + { CDF15(17248, 26058, 28904, 30608, 31305, 31877, 32126, 32321, + 32394, 32464, 32516, 32560, 32576, 32593, 32622) }, + { CDF15(14738, 21678, 25779, 27901, 29024, 30302, 30980, 31843, + 32144, 32413, 32520, 32594, 32622, 32656, 32660) }, + }, .restore_wiener = { + CDF1(11570) + }, .restore_sgrproj = { + CDF1(16855) + }, .restore_switchable = { + CDF2( 9413, 22581) + }, .delta_q = { + CDF3(28160, 32120, 32677) + }, .delta_lf = { + { CDF3(28160, 32120, 32677) }, + { CDF3(28160, 32120, 32677) }, + { CDF3(28160, 32120, 32677) }, + { CDF3(28160, 32120, 32677) }, + { CDF3(28160, 32120, 32677) }, + }, .motion_mode = { + [BS_8x8] = { CDF2( 7651, 24760) }, + [BS_8x16] = { CDF2( 4738, 24765) }, + [BS_8x32] = { CDF2(28799, 31390) }, + [BS_16x8] = { CDF2( 5391, 25528) }, + [BS_16x16] = { CDF2(19419, 26810) }, + [BS_16x32] = { CDF2( 5123, 23606) }, + [BS_16x64] = { CDF2(28973, 31594) }, + [BS_32x8] = { CDF2(26431, 30774) }, + [BS_32x16] = { CDF2(11606, 24308) }, + [BS_32x32] = { CDF2(26260, 29116) }, + [BS_32x64] = { CDF2(20360, 28062) }, + [BS_64x16] = { CDF2(29742, 31203) }, + [BS_64x32] = { CDF2(21679, 26830) }, + [BS_64x64] = { CDF2(29516, 30701) }, + [BS_64x128] = { CDF2(28898, 30397) }, + [BS_128x64] = { CDF2(30878, 31335) }, + [BS_128x128] = { CDF2(32507, 32558) }, + }, .obmc = { + [BS_8x8] = { CDF1(10437) }, + [BS_8x16] = { CDF1( 9371) }, + [BS_8x32] = { CDF1(23664) }, + [BS_16x8] = { CDF1( 9301) }, + [BS_16x16] = { CDF1(17432) }, + [BS_16x32] = { CDF1(14423) }, + [BS_16x64] = { CDF1(24008) }, + [BS_32x8] = { CDF1(20901) }, + [BS_32x16] = { CDF1(15142) }, + [BS_32x32] = { CDF1(25817) }, + [BS_32x64] = { CDF1(22823) }, + [BS_64x16] = { CDF1(26879) }, + [BS_64x32] = { CDF1(22083) }, + [BS_64x64] = { CDF1(30128) }, + [BS_64x128] = { CDF1(31014) }, + [BS_128x64] = { CDF1(31560) }, + [BS_128x128] = { CDF1(32638) }, + }, .pal_y = { + { { CDF1(31676) }, { CDF1( 3419) }, { CDF1( 1261) } }, + { { CDF1(31912) }, { CDF1( 2859) }, { CDF1( 980) } }, + { { CDF1(31823) }, { CDF1( 3400) }, { CDF1( 781) } }, + { { CDF1(32030) }, { CDF1( 3561) }, { CDF1( 904) } }, + { { CDF1(32309) }, { CDF1( 7337) }, { CDF1( 1462) } }, + { { CDF1(32265) }, { CDF1( 4015) }, { CDF1( 1521) } }, + { { CDF1(32450) }, { CDF1( 7946) }, { CDF1( 129) } }, + }, .pal_sz = { + { + { CDF6( 7952, 13000, 18149, 21478, 25527, 29241) }, + { CDF6( 7139, 11421, 16195, 19544, 23666, 28073) }, + { CDF6( 7788, 12741, 17325, 20500, 24315, 28530) }, + { CDF6( 8271, 14064, 18246, 21564, 25071, 28533) }, + { CDF6(12725, 19180, 21863, 24839, 27535, 30120) }, + { CDF6( 9711, 14888, 16923, 21052, 25661, 27875) }, + { CDF6(14940, 20797, 21678, 24186, 27033, 28999) }, }, { - { CDF7(21442, 23288, 24758, 26246, 27649, 28980, 30563) }, - { CDF7( 5863, 14933, 17552, 20668, 23683, 26411, 29273) }, - { CDF7( 3415, 25810, 26877, 27990, 29223, 30394, 31618) }, - { CDF7(17965, 20084, 22232, 23974, 26274, 28402, 30390) }, - { CDF7(31190, 31329, 31516, 31679, 31825, 32026, 32322) }, + { CDF6( 8713, 19979, 27128, 29609, 31331, 32272) }, + { CDF6( 5839, 15573, 23581, 26947, 29848, 31700) }, + { CDF6( 4426, 11260, 17999, 21483, 25863, 29430) }, + { CDF6( 3228, 9464, 14993, 18089, 22523, 27420) }, + { CDF6( 3768, 8886, 13091, 17852, 22495, 27207) }, + { CDF6( 2464, 8451, 12861, 21632, 25525, 28555) }, + { CDF6( 1269, 5435, 10433, 18963, 21700, 25865) }, + }, + }, .pal_uv = { + { CDF1(32461) }, { CDF1(21488) }, + }, .color_map = { + { /* y */ + { + { CDF1(28710) }, { CDF1(16384) }, { CDF1(10553) }, + { CDF1(27036) }, { CDF1(31603) }, + }, { + { CDF2(27877, 30490) }, { CDF2(11532, 25697) }, + { CDF2( 6544, 30234) }, { CDF2(23018, 28072) }, + { CDF2(31915, 32385) }, + }, { + { CDF3(25572, 28046, 30045) }, + { CDF3( 9478, 21590, 27256) }, + { CDF3( 7248, 26837, 29824) }, + { CDF3(19167, 24486, 28349) }, + { CDF3(31400, 31825, 32250) }, + }, { + { CDF4(24779, 26955, 28576, 30282) }, + { CDF4( 8669, 20364, 24073, 28093) }, + { CDF4( 4255, 27565, 29377, 31067) }, + { CDF4(19864, 23674, 26716, 29530) }, + { CDF4(31646, 31893, 32147, 32426) }, + }, { + { CDF5(23132, 25407, 26970, 28435, 30073) }, + { CDF5( 7443, 17242, 20717, 24762, 27982) }, + { CDF5( 6300, 24862, 26944, 28784, 30671) }, + { CDF5(18916, 22895, 25267, 27435, 29652) }, + { CDF5(31270, 31550, 31808, 32059, 32353) }, + }, { + { CDF6(23105, 25199, 26464, 27684, 28931, 30318) }, + { CDF6( 6950, 15447, 18952, 22681, 25567, 28563) }, + { CDF6( 7560, 23474, 25490, 27203, 28921, 30708) }, + { CDF6(18544, 22373, 24457, 26195, 28119, 30045) }, + { CDF6(31198, 31451, 31670, 31882, 32123, 32391) }, + }, { + { CDF7(21689, 23883, 25163, 26352, 27506, 28827, 30195) }, + { CDF7( 6892, 15385, 17840, 21606, 24287, 26753, 29204) }, + { CDF7( 5651, 23182, 25042, 26518, 27982, 29392, 30900) }, + { CDF7(19349, 22578, 24418, 25994, 27524, 29031, 30448) }, + { CDF7(31028, 31270, 31504, 31705, 31927, 32153, 32392) }, + }, + }, { /* uv */ + { + { CDF1(29089) }, { CDF1(16384) }, { CDF1( 8713) }, + { CDF1(29257) }, { CDF1(31610) }, + }, { + { CDF2(25257, 29145) }, { CDF2(12287, 27293) }, + { CDF2( 7033, 27960) }, { CDF2(20145, 25405) }, + { CDF2(30608, 31639) }, + }, { + { CDF3(24210, 27175, 29903) }, + { CDF3( 9888, 22386, 27214) }, + { CDF3( 5901, 26053, 29293) }, + { CDF3(18318, 22152, 28333) }, + { CDF3(30459, 31136, 31926) }, + }, { + { CDF4(22980, 25479, 27781, 29986) }, + { CDF4( 8413, 21408, 24859, 28874) }, + { CDF4( 2257, 29449, 30594, 31598) }, + { CDF4(19189, 21202, 25915, 28620) }, + { CDF4(31844, 32044, 32281, 32518) }, + }, { + { CDF5(22217, 24567, 26637, 28683, 30548) }, + { CDF5( 7307, 16406, 19636, 24632, 28424) }, + { CDF5( 4441, 25064, 26879, 28942, 30919) }, + { CDF5(17210, 20528, 23319, 26750, 29582) }, + { CDF5(30674, 30953, 31396, 31735, 32207) }, + }, { + { CDF6(21239, 23168, 25044, 26962, 28705, 30506) }, + { CDF6( 6545, 15012, 18004, 21817, 25503, 28701) }, + { CDF6( 3448, 26295, 27437, 28704, 30126, 31442) }, + { CDF6(15889, 18323, 21704, 24698, 26976, 29690) }, + { CDF6(30988, 31204, 31479, 31734, 31983, 32325) }, + }, { + { CDF7(21442, 23288, 24758, 26246, 27649, 28980, 30563) }, + { CDF7( 5863, 14933, 17552, 20668, 23683, 26411, 29273) }, + { CDF7( 3415, 25810, 26877, 27990, 29223, 30394, 31618) }, + { CDF7(17965, 20084, 22232, 23974, 26274, 28402, 30390) }, + { CDF7(31190, 31329, 31516, 31679, 31825, 32026, 32322) }, + }, }, + }, .intrabc = { + CDF1(30531) + }, + }, .mv = { + .comp = { + .classes = { + CDF10(28672, 30976, 31858, 32320, 32551, + 32656, 32740, 32757, 32762, 32767) + }, .class0 = { + CDF1(27648) + }, .classN = { + { CDF1(17408) }, { CDF1(17920) }, { CDF1(18944) }, + { CDF1(20480) }, { CDF1(22528) }, { CDF1(24576) }, + { CDF1(28672) }, { CDF1(29952) }, { CDF1(29952) }, + { CDF1(30720) }, + }, .class0_fp = { + { CDF3(16384, 24576, 26624) }, + { CDF3(12288, 21248, 24128) }, + }, .classN_fp = { + CDF3( 8192, 17408, 21248) + }, .class0_hp = { + CDF1(20480) + }, .classN_hp = { + CDF1(16384) + }, .sign = { + CDF1(16384) + }, + }, .joint = { + CDF3( 4096, 11264, 19328) + }, + }, .kfym = { + { + { CDF12(15588, 17027, 19338, 20218, 20682, 21110, + 21825, 23244, 24189, 28165, 29093, 30466) }, + { CDF12(12016, 18066, 19516, 20303, 20719, 21444, + 21888, 23032, 24434, 28658, 30172, 31409) }, + { CDF12(10052, 10771, 22296, 22788, 23055, 23239, + 24133, 25620, 26160, 29336, 29929, 31567) }, + { CDF12(14091, 15406, 16442, 18808, 19136, 19546, + 19998, 22096, 24746, 29585, 30958, 32462) }, + { CDF12(12122, 13265, 15603, 16501, 18609, 20033, + 22391, 25583, 26437, 30261, 31073, 32475) }, + }, { + { CDF12(10023, 19585, 20848, 21440, 21832, 22760, + 23089, 24023, 25381, 29014, 30482, 31436) }, + { CDF12( 5983, 24099, 24560, 24886, 25066, 25795, + 25913, 26423, 27610, 29905, 31276, 31794) }, + { CDF12( 7444, 12781, 20177, 20728, 21077, 21607, + 22170, 23405, 24469, 27915, 29090, 30492) }, + { CDF12( 8537, 14689, 15432, 17087, 17408, 18172, + 18408, 19825, 24649, 29153, 31096, 32210) }, + { CDF12( 7543, 14231, 15496, 16195, 17905, 20717, + 21984, 24516, 26001, 29675, 30981, 31994) }, + }, { + { CDF12(12613, 13591, 21383, 22004, 22312, 22577, + 23401, 25055, 25729, 29538, 30305, 32077) }, + { CDF12( 9687, 13470, 18506, 19230, 19604, 20147, + 20695, 22062, 23219, 27743, 29211, 30907) }, + { CDF12( 6183, 6505, 26024, 26252, 26366, 26434, + 27082, 28354, 28555, 30467, 30794, 32086) }, + { CDF12(10718, 11734, 14954, 17224, 17565, 17924, + 18561, 21523, 23878, 28975, 30287, 32252) }, + { CDF12( 9194, 9858, 16501, 17263, 18424, 19171, + 21563, 25961, 26561, 30072, 30737, 32463) }, + }, { + { CDF12(12602, 14399, 15488, 18381, 18778, 19315, + 19724, 21419, 25060, 29696, 30917, 32409) }, + { CDF12( 8203, 13821, 14524, 17105, 17439, 18131, + 18404, 19468, 25225, 29485, 31158, 32342) }, + { CDF12( 8451, 9731, 15004, 17643, 18012, 18425, + 19070, 21538, 24605, 29118, 30078, 32018) }, + { CDF12( 7714, 9048, 9516, 16667, 16817, 16994, + 17153, 18767, 26743, 30389, 31536, 32528) }, + { CDF12( 8843, 10280, 11496, 15317, 16652, 17943, + 19108, 22718, 25769, 29953, 30983, 32485) }, + }, { + { CDF12(12578, 13671, 15979, 16834, 19075, 20913, + 22989, 25449, 26219, 30214, 31150, 32477) }, + { CDF12( 9563, 13626, 15080, 15892, 17756, 20863, + 22207, 24236, 25380, 29653, 31143, 32277) }, + { CDF12( 8356, 8901, 17616, 18256, 19350, 20106, + 22598, 25947, 26466, 29900, 30523, 32261) }, + { CDF12(10835, 11815, 13124, 16042, 17018, 18039, + 18947, 22753, 24615, 29489, 30883, 32482) }, + { CDF12( 7618, 8288, 9859, 10509, 15386, 18657, + 22903, 28776, 29180, 31355, 31802, 32593) }, }, - }, .intrabc = { - CDF1(30531) - }, -}; - -static const CdfMvComponent default_mv_component_cdf = { - .classes = { - CDF10(28672, 30976, 31858, 32320, 32551, - 32656, 32740, 32757, 32762, 32767) - }, .class0 = { - CDF1(27648) - }, .classN = { - { CDF1(17408) }, { CDF1(17920) }, { CDF1(18944) }, - { CDF1(20480) }, { CDF1(22528) }, { CDF1(24576) }, - { CDF1(28672) }, { CDF1(29952) }, { CDF1(29952) }, - { CDF1(30720) }, - }, .class0_fp = { - { CDF3(16384, 24576, 26624) }, - { CDF3(12288, 21248, 24128) }, - }, .classN_fp = { - CDF3( 8192, 17408, 21248) - }, .class0_hp = { - CDF1(20480) - }, .classN_hp = { - CDF1(16384) - }, .sign = { - CDF1(16384) - }, -}; - -static const uint16_t ALIGN(default_mv_joint_cdf[N_MV_JOINTS], 8) = { - CDF3( 4096, 11264, 19328) -}; - -static const uint16_t ALIGN(default_kf_y_mode_cdf[5][5][N_INTRA_PRED_MODES + 3], 32) = { - { - { CDF12(15588, 17027, 19338, 20218, 20682, 21110, - 21825, 23244, 24189, 28165, 29093, 30466) }, - { CDF12(12016, 18066, 19516, 20303, 20719, 21444, - 21888, 23032, 24434, 28658, 30172, 31409) }, - { CDF12(10052, 10771, 22296, 22788, 23055, 23239, - 24133, 25620, 26160, 29336, 29929, 31567) }, - { CDF12(14091, 15406, 16442, 18808, 19136, 19546, - 19998, 22096, 24746, 29585, 30958, 32462) }, - { CDF12(12122, 13265, 15603, 16501, 18609, 20033, - 22391, 25583, 26437, 30261, 31073, 32475) }, - }, { - { CDF12(10023, 19585, 20848, 21440, 21832, 22760, - 23089, 24023, 25381, 29014, 30482, 31436) }, - { CDF12( 5983, 24099, 24560, 24886, 25066, 25795, - 25913, 26423, 27610, 29905, 31276, 31794) }, - { CDF12( 7444, 12781, 20177, 20728, 21077, 21607, - 22170, 23405, 24469, 27915, 29090, 30492) }, - { CDF12( 8537, 14689, 15432, 17087, 17408, 18172, - 18408, 19825, 24649, 29153, 31096, 32210) }, - { CDF12( 7543, 14231, 15496, 16195, 17905, 20717, - 21984, 24516, 26001, 29675, 30981, 31994) }, - }, { - { CDF12(12613, 13591, 21383, 22004, 22312, 22577, - 23401, 25055, 25729, 29538, 30305, 32077) }, - { CDF12( 9687, 13470, 18506, 19230, 19604, 20147, - 20695, 22062, 23219, 27743, 29211, 30907) }, - { CDF12( 6183, 6505, 26024, 26252, 26366, 26434, - 27082, 28354, 28555, 30467, 30794, 32086) }, - { CDF12(10718, 11734, 14954, 17224, 17565, 17924, - 18561, 21523, 23878, 28975, 30287, 32252) }, - { CDF12( 9194, 9858, 16501, 17263, 18424, 19171, - 21563, 25961, 26561, 30072, 30737, 32463) }, - }, { - { CDF12(12602, 14399, 15488, 18381, 18778, 19315, - 19724, 21419, 25060, 29696, 30917, 32409) }, - { CDF12( 8203, 13821, 14524, 17105, 17439, 18131, - 18404, 19468, 25225, 29485, 31158, 32342) }, - { CDF12( 8451, 9731, 15004, 17643, 18012, 18425, - 19070, 21538, 24605, 29118, 30078, 32018) }, - { CDF12( 7714, 9048, 9516, 16667, 16817, 16994, - 17153, 18767, 26743, 30389, 31536, 32528) }, - { CDF12( 8843, 10280, 11496, 15317, 16652, 17943, - 19108, 22718, 25769, 29953, 30983, 32485) }, - }, { - { CDF12(12578, 13671, 15979, 16834, 19075, 20913, - 22989, 25449, 26219, 30214, 31150, 32477) }, - { CDF12( 9563, 13626, 15080, 15892, 17756, 20863, - 22207, 24236, 25380, 29653, 31143, 32277) }, - { CDF12( 8356, 8901, 17616, 18256, 19350, 20106, - 22598, 25947, 26466, 29900, 30523, 32261) }, - { CDF12(10835, 11815, 13124, 16042, 17018, 18039, - 18947, 22753, 24615, 29489, 30883, 32482) }, - { CDF12( 7618, 8288, 9859, 10509, 15386, 18657, - 22903, 28776, 29180, 31355, 31802, 32593) }, }, }; -static const CdfCoefContext av1_default_coef_cdf[4] = { +static const CdfCoefContext default_coef_cdf[4] = { [0] = { .skip = { { @@ -3951,10 +3958,8 @@ void dav1d_cdf_thread_update(const Dav1dFrameHeader *const hdr, { #define update_cdf_1d(n1d, name) \ do { \ - memcpy(dst->name, src->name, sizeof(dst->name)); \ dst->name[n1d] = 0; \ } while (0) - #define update_cdf_2d(n1d, n2d, name) \ for (int j = 0; j < (n1d); j++) update_cdf_1d(n2d, name[j]) #define update_cdf_3d(n1d, n2d, n3d, name) \ @@ -3962,29 +3967,8 @@ void dav1d_cdf_thread_update(const Dav1dFrameHeader *const hdr, #define update_cdf_4d(n1d, n2d, n3d, n4d, name) \ for (int l = 0; l < (n1d); l++) update_cdf_3d(n2d, n3d, n4d, name[l]) -#define update_bit_0d(name) \ - do { \ - dst->name[0] = src->name[0]; \ - dst->name[1] = 0; \ - } while (0) - -#define update_bit_1d(n1d, name) \ - for (int i = 0; i < (n1d); i++) update_bit_0d(name[i]) -#define update_bit_2d(n1d, n2d, name) \ - for (int j = 0; j < (n1d); j++) update_bit_1d(n2d, name[j]) -#define update_bit_3d(n1d, n2d, n3d, name) \ - for (int k = 0; k < (n1d); k++) update_bit_2d(n2d, n3d, name[k]) + memcpy(dst, src, offsetof(CdfContext, m.intrabc)); - update_bit_1d(N_BS_SIZES, m.use_filter_intra); - update_cdf_1d(4, m.filter_intra); - update_cdf_3d(2, N_INTRA_PRED_MODES, N_UV_INTRA_PRED_MODES - 1 - !k, m.uv_mode); - update_cdf_2d(8, 6, m.angle_delta); - update_cdf_3d(N_TX_SIZES - 1, 3, imin(k + 1, 2), m.txsz); - update_cdf_3d(2, N_INTRA_PRED_MODES, 6, m.txtp_intra1); - update_cdf_3d(3, N_INTRA_PRED_MODES, 4, m.txtp_intra2); - update_bit_1d(3, m.skip); - update_cdf_3d(N_BL_LEVELS, 4, dav1d_partition_type_count[k], m.partition); - update_bit_2d(N_TX_SIZES, 13, coef.skip); update_cdf_3d(2, 2, 4, coef.eob_bin_16); update_cdf_3d(2, 2, 5, coef.eob_bin_32); update_cdf_3d(2, 2, 6, coef.eob_bin_64); @@ -3992,106 +3976,104 @@ void dav1d_cdf_thread_update(const Dav1dFrameHeader *const hdr, update_cdf_3d(2, 2, 8, coef.eob_bin_256); update_cdf_2d(2, 9, coef.eob_bin_512); update_cdf_2d(2, 10, coef.eob_bin_1024); - update_bit_3d(N_TX_SIZES, 2, 11 /*22*/, coef.eob_hi_bit); update_cdf_4d(N_TX_SIZES, 2, 4, 2, coef.eob_base_tok); update_cdf_4d(N_TX_SIZES, 2, 41 /*42*/, 3, coef.base_tok); - update_bit_2d(2, 3, coef.dc_sign); update_cdf_4d(4, 2, 21, 3, coef.br_tok); - update_cdf_2d(3, DAV1D_MAX_SEGMENTS - 1, m.seg_id); - update_cdf_1d(7, m.cfl_sign); + update_cdf_4d(N_TX_SIZES, 2, 11 /*22*/, 1, coef.eob_hi_bit); + update_cdf_3d(N_TX_SIZES, 13, 1, coef.skip); + update_cdf_3d(2, 3, 1, coef.dc_sign); + + update_cdf_3d(2, N_INTRA_PRED_MODES, N_UV_INTRA_PRED_MODES - 1 - !k, m.uv_mode); + update_cdf_2d(4, N_PARTITIONS - 3, m.partition[BL_128X128]); + for (int k = BL_64X64; k < BL_8X8; k++) + update_cdf_2d(4, N_PARTITIONS - 1, m.partition[k]); + update_cdf_2d(4, N_SUB8X8_PARTITIONS - 1, m.partition[BL_8X8]); update_cdf_2d(6, 15, m.cfl_alpha); - update_bit_0d(m.restore_wiener); - update_bit_0d(m.restore_sgrproj); - update_cdf_1d(2, m.restore_switchable); - update_cdf_1d(3, m.delta_q); - update_cdf_2d(5, 3, m.delta_lf); - update_bit_2d(7, 3, m.pal_y); - update_bit_1d(2, m.pal_uv); - update_cdf_3d(2, 7, 6, m.pal_sz); - update_cdf_4d(2, 7, 5, k + 1, m.color_map); - update_bit_2d(7, 3, m.txpart); update_cdf_2d(2, 15, m.txtp_inter1); update_cdf_1d(11, m.txtp_inter2); - update_bit_1d(4, m.txtp_inter3); - - if (IS_KEY_OR_INTRA(hdr)) { - update_bit_0d(m.intrabc); + update_cdf_3d(2, N_INTRA_PRED_MODES, 6, m.txtp_intra1); + update_cdf_3d(3, N_INTRA_PRED_MODES, 4, m.txtp_intra2); + update_cdf_1d(7, m.cfl_sign); + update_cdf_2d(8, 6, m.angle_delta); + update_cdf_1d(4, m.filter_intra); + update_cdf_2d(3, DAV1D_MAX_SEGMENTS - 1, m.seg_id); + update_cdf_3d(2, 7, 6, m.pal_sz); + update_cdf_4d(2, 7, 5, k + 1, m.color_map); + update_cdf_3d(N_TX_SIZES - 1, 3, imin(k + 1, 2), m.txsz); + update_cdf_1d(3, m.delta_q); + update_cdf_2d(5, 3, m.delta_lf); + update_cdf_1d(2, m.restore_switchable); + update_cdf_1d(1, m.restore_wiener); + update_cdf_1d(1, m.restore_sgrproj); + update_cdf_2d(4, 1, m.txtp_inter3); + update_cdf_2d(N_BS_SIZES, 1, m.use_filter_intra); + update_cdf_3d(7, 3, 1, m.txpart); + update_cdf_2d(3, 1, m.skip); + update_cdf_3d(7, 3, 1, m.pal_y); + update_cdf_2d(2, 1, m.pal_uv); - update_cdf_1d(N_MV_JOINTS - 1, dmv.joint); - for (int k = 0; k < 2; k++) { - update_cdf_1d(10, dmv.comp[k].classes); - update_bit_0d(dmv.comp[k].class0); - update_bit_1d(10, dmv.comp[k].classN); - update_bit_0d(dmv.comp[k].sign); - } + if (IS_KEY_OR_INTRA(hdr)) return; - } - update_bit_1d(3, m.skip_mode); + memcpy(dst->m.y_mode, src->m.y_mode, + offsetof(CdfContext, kfym) - offsetof(CdfContext, m.y_mode)); + update_cdf_2d(4, N_INTRA_PRED_MODES - 1, m.y_mode); - update_cdf_3d(2, 8, DAV1D_N_SWITCHABLE_FILTERS - 1, m.filter); - update_bit_1d(6, m.newmv_mode); - update_bit_1d(2, m.globalmv_mode); - update_bit_1d(6, m.refmv_mode); - update_bit_1d(3, m.drl_bit); - update_cdf_2d(8, N_COMP_INTER_PRED_MODES - 1, m.comp_inter_mode); - update_bit_1d(4, m.intra); - update_bit_1d(5, m.comp); - update_bit_1d(5, m.comp_dir); - update_bit_1d(6, m.jnt_comp); - update_bit_1d(6, m.mask_comp); - update_bit_1d(9, m.wedge_comp); update_cdf_2d(9, 15, m.wedge_idx); - update_bit_2d(6, 3, m.ref); - update_bit_2d(3, 3, m.comp_fwd_ref); - update_bit_2d(2, 3, m.comp_bwd_ref); - update_bit_2d(3, 3, m.comp_uni_ref); - update_bit_1d(3, m.seg_pred); - update_bit_1d(4, m.interintra); - update_bit_1d(7, m.interintra_wedge); + update_cdf_2d(8, N_COMP_INTER_PRED_MODES - 1, m.comp_inter_mode); + update_cdf_3d(2, 8, DAV1D_N_SWITCHABLE_FILTERS - 1, m.filter); update_cdf_2d(4, 3, m.interintra_mode); update_cdf_2d(N_BS_SIZES, 2, m.motion_mode); - update_bit_1d(N_BS_SIZES, m.obmc); + update_cdf_2d(3, 1, m.skip_mode); + update_cdf_2d(6, 1, m.newmv_mode); + update_cdf_2d(2, 1, m.globalmv_mode); + update_cdf_2d(6, 1, m.refmv_mode); + update_cdf_2d(3, 1, m.drl_bit); + update_cdf_2d(4, 1, m.intra); + update_cdf_2d(5, 1, m.comp); + update_cdf_2d(5, 1, m.comp_dir); + update_cdf_2d(6, 1, m.jnt_comp); + update_cdf_2d(6, 1, m.mask_comp); + update_cdf_2d(9, 1, m.wedge_comp); + update_cdf_3d(6, 3, 1, m.ref); + update_cdf_3d(3, 3, 1, m.comp_fwd_ref); + update_cdf_3d(2, 3, 1, m.comp_bwd_ref); + update_cdf_3d(3, 3, 1, m.comp_uni_ref); + update_cdf_2d(3, 1, m.seg_pred); + update_cdf_2d(4, 1, m.interintra); + update_cdf_2d(7, 1, m.interintra_wedge); + update_cdf_2d(N_BS_SIZES, 1, m.obmc); - update_cdf_1d(N_MV_JOINTS - 1, mv.joint); for (int k = 0; k < 2; k++) { update_cdf_1d(10, mv.comp[k].classes); - update_bit_0d(mv.comp[k].class0); - update_bit_1d(10, mv.comp[k].classN); + update_cdf_1d(1, mv.comp[k].sign); + update_cdf_1d(1, mv.comp[k].class0); update_cdf_2d(2, 3, mv.comp[k].class0_fp); + update_cdf_1d(1, mv.comp[k].class0_hp); + update_cdf_2d(10, 1, mv.comp[k].classN); update_cdf_1d(3, mv.comp[k].classN_fp); - update_bit_0d(mv.comp[k].class0_hp); - update_bit_0d(mv.comp[k].classN_hp); - update_bit_0d(mv.comp[k].sign); + update_cdf_1d(1, mv.comp[k].classN_hp); } + update_cdf_1d(N_MV_JOINTS - 1, mv.joint); } /* * CDF threading wrappers. */ -static inline int get_qcat_idx(const int q) { - if (q <= 20) return 0; - if (q <= 60) return 1; - if (q <= 120) return 2; - return 3; -} - -void dav1d_cdf_thread_init_static(CdfThreadContext *const cdf, const int qidx) { +void dav1d_cdf_thread_init_static(CdfThreadContext *const cdf, const unsigned qidx) { cdf->ref = NULL; - cdf->data.qcat = get_qcat_idx(qidx); + cdf->data.qcat = (qidx > 20) + (qidx > 60) + (qidx > 120); } void dav1d_cdf_thread_copy(CdfContext *const dst, const CdfThreadContext *const src) { if (src->ref) { memcpy(dst, src->data.cdf, sizeof(*dst)); } else { - dst->m = av1_default_cdf; - memcpy(dst->kfym, default_kf_y_mode_cdf, sizeof(default_kf_y_mode_cdf)); - dst->coef = av1_default_coef_cdf[src->data.qcat]; - memcpy(dst->mv.joint, default_mv_joint_cdf, sizeof(default_mv_joint_cdf)); - memcpy(dst->dmv.joint, default_mv_joint_cdf, sizeof(default_mv_joint_cdf)); - dst->mv.comp[0] = dst->mv.comp[1] = dst->dmv.comp[0] = dst->dmv.comp[1] = - default_mv_component_cdf; + dst->coef = default_coef_cdf[src->data.qcat]; + memcpy(&dst->m, &default_cdf.m, + offsetof(CdfDefaultContext, mv.joint)); + memcpy(&dst->mv.comp[1], &default_cdf.mv.comp, + sizeof(default_cdf) - offsetof(CdfDefaultContext, mv.comp)); } } diff --git a/third_party/dav1d/src/cdf.h b/third_party/dav1d/src/cdf.h index 4b30474baa..c9b516dc72 100644 --- a/third_party/dav1d/src/cdf.h +++ b/third_party/dav1d/src/cdf.h @@ -34,12 +34,10 @@ #include "src/ref.h" #include "src/thread_data.h" -/* Buffers padded to [8] or [16] for SIMD where needed. */ +/* Buffers padded to [4]/[8]/[16] for SIMD where needed. */ typedef struct CdfModeContext { - ALIGN(uint16_t y_mode[4][N_INTRA_PRED_MODES + 3], 32); ALIGN(uint16_t uv_mode[2][N_INTRA_PRED_MODES][N_UV_INTRA_PRED_MODES + 2], 32); - ALIGN(uint16_t wedge_idx[9][16], 32); ALIGN(uint16_t partition[N_BL_LEVELS][4][N_PARTITIONS + 6], 32); ALIGN(uint16_t cfl_alpha[6][16], 32); ALIGN(uint16_t txtp_inter1[2][16], 32); @@ -49,23 +47,33 @@ typedef struct CdfModeContext { ALIGN(uint16_t cfl_sign[8], 16); ALIGN(uint16_t angle_delta[8][8], 16); ALIGN(uint16_t filter_intra[5 + 3], 16); - ALIGN(uint16_t comp_inter_mode[8][N_COMP_INTER_PRED_MODES], 16); ALIGN(uint16_t seg_id[3][DAV1D_MAX_SEGMENTS], 16); ALIGN(uint16_t pal_sz[2][7][7 + 1], 16); ALIGN(uint16_t color_map[2][7][5][8], 16); - ALIGN(uint16_t filter[2][8][DAV1D_N_SWITCHABLE_FILTERS + 1], 8); ALIGN(uint16_t txsz[N_TX_SIZES - 1][3][4], 8); - ALIGN(uint16_t motion_mode[N_BS_SIZES][3 + 1], 8); ALIGN(uint16_t delta_q[4], 8); ALIGN(uint16_t delta_lf[5][4], 8); - ALIGN(uint16_t interintra_mode[4][4], 8); ALIGN(uint16_t restore_switchable[3 + 1], 8); ALIGN(uint16_t restore_wiener[2], 4); ALIGN(uint16_t restore_sgrproj[2], 4); - ALIGN(uint16_t interintra[7][2], 4); - ALIGN(uint16_t interintra_wedge[7][2], 4); ALIGN(uint16_t txtp_inter3[4][2], 4); ALIGN(uint16_t use_filter_intra[N_BS_SIZES][2], 4); + ALIGN(uint16_t txpart[7][3][2], 4); + ALIGN(uint16_t skip[3][2], 4); + ALIGN(uint16_t pal_y[7][3][2], 4); + ALIGN(uint16_t pal_uv[2][2], 4); + + /* key/intra */ + ALIGN(uint16_t intrabc[2], 4); + + /* inter/switch */ + ALIGN(uint16_t y_mode[4][N_INTRA_PRED_MODES + 3], 32); + ALIGN(uint16_t wedge_idx[9][16], 32); + ALIGN(uint16_t comp_inter_mode[8][N_COMP_INTER_PRED_MODES], 16); + ALIGN(uint16_t filter[2][8][DAV1D_N_SWITCHABLE_FILTERS + 1], 8); + ALIGN(uint16_t interintra_mode[4][4], 8); + ALIGN(uint16_t motion_mode[N_BS_SIZES][3 + 1], 8); + ALIGN(uint16_t skip_mode[3][2], 4); ALIGN(uint16_t newmv_mode[6][2], 4); ALIGN(uint16_t globalmv_mode[2][2], 4); ALIGN(uint16_t refmv_mode[6][2], 4); @@ -80,14 +88,10 @@ typedef struct CdfModeContext { ALIGN(uint16_t comp_fwd_ref[3][3][2], 4); ALIGN(uint16_t comp_bwd_ref[2][3][2], 4); ALIGN(uint16_t comp_uni_ref[3][3][2], 4); - ALIGN(uint16_t txpart[7][3][2], 4); - ALIGN(uint16_t skip[3][2], 4); - ALIGN(uint16_t skip_mode[3][2], 4); ALIGN(uint16_t seg_pred[3][2], 4); + ALIGN(uint16_t interintra[7][2], 4); + ALIGN(uint16_t interintra_wedge[7][2], 4); ALIGN(uint16_t obmc[N_BS_SIZES][2], 4); - ALIGN(uint16_t pal_y[7][3][2], 4); - ALIGN(uint16_t pal_uv[2][2], 4); - ALIGN(uint16_t intrabc[2], 4); } CdfModeContext; typedef struct CdfCoefContext { @@ -108,13 +112,13 @@ typedef struct CdfCoefContext { typedef struct CdfMvComponent { ALIGN(uint16_t classes[11 + 5], 32); + ALIGN(uint16_t sign[2], 4); + ALIGN(uint16_t class0[2], 4); ALIGN(uint16_t class0_fp[2][4], 8); - ALIGN(uint16_t classN_fp[4], 8); ALIGN(uint16_t class0_hp[2], 4); - ALIGN(uint16_t classN_hp[2], 4); - ALIGN(uint16_t class0[2], 4); ALIGN(uint16_t classN[10][2], 4); - ALIGN(uint16_t sign[2], 4); + ALIGN(uint16_t classN_fp[4], 8); + ALIGN(uint16_t classN_hp[2], 4); } CdfMvComponent; typedef struct CdfMvContext { @@ -123,10 +127,10 @@ typedef struct CdfMvContext { } CdfMvContext; typedef struct CdfContext { + CdfCoefContext coef; CdfModeContext m; + CdfMvContext mv; ALIGN(uint16_t kfym[5][5][N_INTRA_PRED_MODES + 3], 32); - CdfCoefContext coef; - CdfMvContext mv, dmv; } CdfContext; typedef struct CdfThreadContext { @@ -138,7 +142,7 @@ typedef struct CdfThreadContext { atomic_uint *progress; } CdfThreadContext; -void dav1d_cdf_thread_init_static(CdfThreadContext *cdf, int qidx); +void dav1d_cdf_thread_init_static(CdfThreadContext *cdf, unsigned qidx); int dav1d_cdf_thread_alloc(Dav1dContext *c, CdfThreadContext *cdf, const int have_frame_mt); void dav1d_cdf_thread_copy(CdfContext *dst, const CdfThreadContext *src); diff --git a/third_party/dav1d/src/decode.c b/third_party/dav1d/src/decode.c index eed9dfb756..7427c35592 100644 --- a/third_party/dav1d/src/decode.c +++ b/third_party/dav1d/src/decode.c @@ -73,42 +73,29 @@ static void init_quant_tables(const Dav1dSequenceHeader *const seq_hdr, } } -static int read_mv_component_diff(Dav1dTaskContext *const t, +static int read_mv_component_diff(MsacContext *const msac, CdfMvComponent *const mv_comp, - const int have_fp) + const int mv_prec) { - Dav1dTileState *const ts = t->ts; - const Dav1dFrameContext *const f = t->f; - const int have_hp = f->frame_hdr->hp; - const int sign = dav1d_msac_decode_bool_adapt(&ts->msac, mv_comp->sign); - const int cl = dav1d_msac_decode_symbol_adapt16(&ts->msac, - mv_comp->classes, 10); - int up, fp, hp; + const int sign = dav1d_msac_decode_bool_adapt(msac, mv_comp->sign); + const int cl = dav1d_msac_decode_symbol_adapt16(msac, mv_comp->classes, 10); + int up, fp = 3, hp = 1; if (!cl) { - up = dav1d_msac_decode_bool_adapt(&ts->msac, mv_comp->class0); - if (have_fp) { - fp = dav1d_msac_decode_symbol_adapt4(&ts->msac, - mv_comp->class0_fp[up], 3); - hp = have_hp ? dav1d_msac_decode_bool_adapt(&ts->msac, - mv_comp->class0_hp) : 1; - } else { - fp = 3; - hp = 1; + up = dav1d_msac_decode_bool_adapt(msac, mv_comp->class0); + if (mv_prec >= 0) { // !force_integer_mv + fp = dav1d_msac_decode_symbol_adapt4(msac, mv_comp->class0_fp[up], 3); + if (mv_prec > 0) // allow_high_precision_mv + hp = dav1d_msac_decode_bool_adapt(msac, mv_comp->class0_hp); } } else { up = 1 << cl; for (int n = 0; n < cl; n++) - up |= dav1d_msac_decode_bool_adapt(&ts->msac, - mv_comp->classN[n]) << n; - if (have_fp) { - fp = dav1d_msac_decode_symbol_adapt4(&ts->msac, - mv_comp->classN_fp, 3); - hp = have_hp ? dav1d_msac_decode_bool_adapt(&ts->msac, - mv_comp->classN_hp) : 1; - } else { - fp = 3; - hp = 1; + up |= dav1d_msac_decode_bool_adapt(msac, mv_comp->classN[n]) << n; + if (mv_prec >= 0) { // !force_integer_mv + fp = dav1d_msac_decode_symbol_adapt4(msac, mv_comp->classN_fp, 3); + if (mv_prec > 0) // allow_high_precision_mv + hp = dav1d_msac_decode_bool_adapt(msac, mv_comp->classN_hp); } } @@ -117,25 +104,16 @@ static int read_mv_component_diff(Dav1dTaskContext *const t, return sign ? -diff : diff; } -static void read_mv_residual(Dav1dTaskContext *const t, mv *const ref_mv, - CdfMvContext *const mv_cdf, const int have_fp) +static void read_mv_residual(Dav1dTileState *const ts, mv *const ref_mv, + const int mv_prec) { - switch (dav1d_msac_decode_symbol_adapt4(&t->ts->msac, t->ts->cdf.mv.joint, - N_MV_JOINTS - 1)) - { - case MV_JOINT_HV: - ref_mv->y += read_mv_component_diff(t, &mv_cdf->comp[0], have_fp); - ref_mv->x += read_mv_component_diff(t, &mv_cdf->comp[1], have_fp); - break; - case MV_JOINT_H: - ref_mv->x += read_mv_component_diff(t, &mv_cdf->comp[1], have_fp); - break; - case MV_JOINT_V: - ref_mv->y += read_mv_component_diff(t, &mv_cdf->comp[0], have_fp); - break; - default: - break; - } + MsacContext *const msac = &ts->msac; + const enum MVJoint mv_joint = + dav1d_msac_decode_symbol_adapt4(msac, ts->cdf.mv.joint, N_MV_JOINTS - 1); + if (mv_joint & MV_JOINT_V) + ref_mv->y += read_mv_component_diff(msac, &ts->cdf.mv.comp[0], mv_prec); + if (mv_joint & MV_JOINT_H) + ref_mv->x += read_mv_component_diff(msac, &ts->cdf.mv.comp[1], mv_prec); } static void read_tx_tree(Dav1dTaskContext *const t, @@ -1001,8 +979,7 @@ static int decode_b(Dav1dTaskContext *const t, const int have_delta_q = f->frame_hdr->delta.q.present && (bs != (f->seq_hdr->sb128 ? BS_128x128 : BS_64x64) || !b->skip); - int8_t prev_delta_lf[4]; - memcpy(prev_delta_lf, ts->last_delta_lf, 4); + uint32_t prev_delta_lf = ts->last_delta_lf.u32; if (have_delta_q) { int delta_q = dav1d_msac_decode_symbol_adapt4(&ts->msac, @@ -1038,8 +1015,8 @@ static int decode_b(Dav1dTaskContext *const t, delta_lf = -delta_lf; delta_lf *= 1 << f->frame_hdr->delta.lf.res_log2; } - ts->last_delta_lf[i] = - iclip(ts->last_delta_lf[i] + delta_lf, -63, 63); + ts->last_delta_lf.i8[i] = + iclip(ts->last_delta_lf.i8[i] + delta_lf, -63, 63); if (have_delta_q && DEBUG_BLOCK_INFO) printf("Post-delta_lf[%d:%d]: r=%d\n", i, delta_lf, ts->msac.rng); @@ -1054,13 +1031,13 @@ static int decode_b(Dav1dTaskContext *const t, init_quant_tables(f->seq_hdr, f->frame_hdr, ts->last_qidx, ts->dqmem); ts->dq = ts->dqmem; } - if (!memcmp(ts->last_delta_lf, (int8_t[4]) { 0, 0, 0, 0 }, 4)) { + if (!ts->last_delta_lf.u32) { // assign frame-wide lf values to this sb ts->lflvl = f->lf.lvl; - } else if (memcmp(ts->last_delta_lf, prev_delta_lf, 4)) { + } else if (ts->last_delta_lf.u32 != prev_delta_lf) { // find sb-specific lf lvl parameters - dav1d_calc_lf_values(ts->lflvlmem, f->frame_hdr, ts->last_delta_lf); ts->lflvl = ts->lflvlmem; + dav1d_calc_lf_values(ts->lflvlmem, f->frame_hdr, ts->last_delta_lf.i8); } } @@ -1324,7 +1301,7 @@ static int decode_b(Dav1dTaskContext *const t, } const union mv ref = b->mv[0]; - read_mv_residual(t, &b->mv[0], &ts->cdf.dmv, 0); + read_mv_residual(ts, &b->mv[0], -1); // clip intrabc motion vector to decoded parts of current tile int border_left = ts->tiling.col_start * 4; @@ -1586,8 +1563,8 @@ static int decode_b(Dav1dTaskContext *const t, break; \ case NEWMV: \ b->mv[idx] = mvstack[b->drl_idx].mv.mv[idx]; \ - read_mv_residual(t, &b->mv[idx], &ts->cdf.mv, \ - !f->frame_hdr->force_integer_mv); \ + const int mv_prec = f->frame_hdr->hp - f->frame_hdr->force_integer_mv; \ + read_mv_residual(ts, &b->mv[idx], mv_prec); \ break; \ } has_subpel_filter = imin(bw4, bh4) == 1 || @@ -1775,8 +1752,8 @@ static int decode_b(Dav1dTaskContext *const t, if (DEBUG_BLOCK_INFO) printf("Post-intermode[%d,drl=%d]: r=%d\n", b->inter_mode, b->drl_idx, ts->msac.rng); - read_mv_residual(t, &b->mv[0], &ts->cdf.mv, - !f->frame_hdr->force_integer_mv); + const int mv_prec = f->frame_hdr->hp - f->frame_hdr->force_integer_mv; + read_mv_residual(ts, &b->mv[0], mv_prec); if (DEBUG_BLOCK_INFO) printf("Post-residualmv[mv=y:%d,x:%d]: r=%d\n", b->mv[0].y, b->mv[0].x, ts->msac.rng); @@ -2495,7 +2472,7 @@ static void setup_tile(Dav1dTileState *const ts, dav1d_cdf_thread_copy(&ts->cdf, &f->in_cdf); ts->last_qidx = f->frame_hdr->quant.yac; - memset(ts->last_delta_lf, 0, sizeof(ts->last_delta_lf)); + ts->last_delta_lf.u32 = 0; dav1d_msac_init(&ts->msac, data, sz, f->frame_hdr->disable_cdf_update); diff --git a/third_party/dav1d/src/internal.h b/third_party/dav1d/src/internal.h index 72f65607ed..96bf409c6c 100644 --- a/third_party/dav1d/src/internal.h +++ b/third_party/dav1d/src/internal.h @@ -303,8 +303,8 @@ struct Dav1dFrameContext { int lr_buf_plane_sz[2]; /* (stride*sbh*4) << sb128 if n_tc > 1, else stride*4 */ int re_sz /* h */; ALIGN(Av1FilterLUT lim_lut, 16); + ALIGN(uint8_t lvl[8 /* seg_id */][4 /* dir */][8 /* ref */][2 /* is_gmv */], 16); int last_sharpness; - uint8_t lvl[8 /* seg_id */][4 /* dir */][8 /* ref */][2 /* is_gmv */]; uint8_t *tx_lpf_right_edge[2]; uint8_t *cdef_line_buf, *lr_line_buf; pixel *cdef_line[2 /* pre, post */][3 /* plane */]; @@ -376,8 +376,11 @@ struct Dav1dTileState { const uint16_t (*dq)[3][2]; int last_qidx; - int8_t last_delta_lf[4]; - uint8_t lflvlmem[8 /* seg_id */][4 /* dir */][8 /* ref */][2 /* is_gmv */]; + union { + int8_t i8[4]; + uint32_t u32; + } last_delta_lf; + ALIGN(uint8_t lflvlmem[8 /* seg_id */][4 /* dir */][8 /* ref */][2 /* is_gmv */], 16); const uint8_t (*lflvl)[4][8][2]; Av1RestorationUnit *lr_ref[3]; diff --git a/third_party/dav1d/src/itx.h b/third_party/dav1d/src/itx.h index d522079907..8ef4f4df48 100644 --- a/third_party/dav1d/src/itx.h +++ b/third_party/dav1d/src/itx.h @@ -39,10 +39,73 @@ void (name)(pixel *dst, ptrdiff_t dst_stride, coef *coeff, int eob \ HIGHBD_DECL_SUFFIX) typedef decl_itx_fn(*itxfm_fn); +#define decl_itx2_fns(w, h, opt) \ +decl_itx_fn(BF(dav1d_inv_txfm_add_dct_dct_##w##x##h, opt)); \ +decl_itx_fn(BF(dav1d_inv_txfm_add_identity_identity_##w##x##h, opt)) + +#define decl_itx12_fns(w, h, opt) \ +decl_itx2_fns(w, h, opt); \ +decl_itx_fn(BF(dav1d_inv_txfm_add_dct_adst_##w##x##h, opt)); \ +decl_itx_fn(BF(dav1d_inv_txfm_add_dct_flipadst_##w##x##h, opt)); \ +decl_itx_fn(BF(dav1d_inv_txfm_add_dct_identity_##w##x##h, opt)); \ +decl_itx_fn(BF(dav1d_inv_txfm_add_adst_dct_##w##x##h, opt)); \ +decl_itx_fn(BF(dav1d_inv_txfm_add_adst_adst_##w##x##h, opt)); \ +decl_itx_fn(BF(dav1d_inv_txfm_add_adst_flipadst_##w##x##h, opt)); \ +decl_itx_fn(BF(dav1d_inv_txfm_add_flipadst_dct_##w##x##h, opt)); \ +decl_itx_fn(BF(dav1d_inv_txfm_add_flipadst_adst_##w##x##h, opt)); \ +decl_itx_fn(BF(dav1d_inv_txfm_add_flipadst_flipadst_##w##x##h, opt)); \ +decl_itx_fn(BF(dav1d_inv_txfm_add_identity_dct_##w##x##h, opt)) + +#define decl_itx16_fns(w, h, opt) \ +decl_itx12_fns(w, h, opt); \ +decl_itx_fn(BF(dav1d_inv_txfm_add_adst_identity_##w##x##h, opt)); \ +decl_itx_fn(BF(dav1d_inv_txfm_add_flipadst_identity_##w##x##h, opt)); \ +decl_itx_fn(BF(dav1d_inv_txfm_add_identity_adst_##w##x##h, opt)); \ +decl_itx_fn(BF(dav1d_inv_txfm_add_identity_flipadst_##w##x##h, opt)) + +#define decl_itx17_fns(w, h, opt) \ +decl_itx16_fns(w, h, opt); \ +decl_itx_fn(BF(dav1d_inv_txfm_add_wht_wht_##w##x##h, opt)) + typedef struct Dav1dInvTxfmDSPContext { itxfm_fn itxfm_add[N_RECT_TX_SIZES][N_TX_TYPES_PLUS_LL]; } Dav1dInvTxfmDSPContext; bitfn_decls(void dav1d_itx_dsp_init, Dav1dInvTxfmDSPContext *c, int bpc); +#define assign_itx_fn(pfx, w, h, type, type_enum, ext) \ + c->itxfm_add[pfx##TX_##w##X##h][type_enum] = \ + BF(dav1d_inv_txfm_add_##type##_##w##x##h, ext) + +#define assign_itx1_fn(pfx, w, h, ext) \ + assign_itx_fn(pfx, w, h, dct_dct, DCT_DCT, ext) + +#define assign_itx2_fn(pfx, w, h, ext) \ + assign_itx1_fn(pfx, w, h, ext); \ + assign_itx_fn(pfx, w, h, identity_identity, IDTX, ext) + +#define assign_itx12_fn(pfx, w, h, ext) \ + assign_itx2_fn(pfx, w, h, ext); \ + assign_itx_fn(pfx, w, h, dct_adst, ADST_DCT, ext); \ + assign_itx_fn(pfx, w, h, dct_flipadst, FLIPADST_DCT, ext); \ + assign_itx_fn(pfx, w, h, dct_identity, H_DCT, ext); \ + assign_itx_fn(pfx, w, h, adst_dct, DCT_ADST, ext); \ + assign_itx_fn(pfx, w, h, adst_adst, ADST_ADST, ext); \ + assign_itx_fn(pfx, w, h, adst_flipadst, FLIPADST_ADST, ext); \ + assign_itx_fn(pfx, w, h, flipadst_dct, DCT_FLIPADST, ext); \ + assign_itx_fn(pfx, w, h, flipadst_adst, ADST_FLIPADST, ext); \ + assign_itx_fn(pfx, w, h, flipadst_flipadst, FLIPADST_FLIPADST, ext); \ + assign_itx_fn(pfx, w, h, identity_dct, V_DCT, ext) + +#define assign_itx16_fn(pfx, w, h, ext) \ + assign_itx12_fn(pfx, w, h, ext); \ + assign_itx_fn(pfx, w, h, adst_identity, H_ADST, ext); \ + assign_itx_fn(pfx, w, h, flipadst_identity, H_FLIPADST, ext); \ + assign_itx_fn(pfx, w, h, identity_adst, V_ADST, ext); \ + assign_itx_fn(pfx, w, h, identity_flipadst, V_FLIPADST, ext) + +#define assign_itx17_fn(pfx, w, h, ext) \ + assign_itx16_fn(pfx, w, h, ext); \ + assign_itx_fn(pfx, w, h, wht_wht, WHT_WHT, ext) + #endif /* DAV1D_SRC_ITX_H */ diff --git a/third_party/dav1d/src/lf_mask.c b/third_party/dav1d/src/lf_mask.c index 062ba67371..09a5c532c4 100644 --- a/third_party/dav1d/src/lf_mask.c +++ b/third_party/dav1d/src/lf_mask.c @@ -436,7 +436,7 @@ static void calc_lf_value(uint8_t (*const lflvl_values)[2], const int base = iclip(iclip(base_lvl + lf_delta, 0, 63) + seg_delta, 0, 63); if (!mr_delta) { - memset(lflvl_values, base, 8 * 2); + memset(lflvl_values, base, sizeof(*lflvl_values) * 8); } else { const int sh = base >= 32; lflvl_values[0][0] = lflvl_values[0][1] = @@ -457,7 +457,7 @@ static inline void calc_lf_value_chroma(uint8_t (*const lflvl_values)[2], const Dav1dLoopfilterModeRefDeltas *const mr_delta) { if (!base_lvl) - memset(lflvl_values, 0, 8 * 2); + memset(lflvl_values, 0, sizeof(*lflvl_values) * 8); else calc_lf_value(lflvl_values, base_lvl, lf_delta, seg_delta, mr_delta); } @@ -469,7 +469,7 @@ void dav1d_calc_lf_values(uint8_t (*const lflvl_values)[4][8][2], const int n_seg = hdr->segmentation.enabled ? 8 : 1; if (!hdr->loopfilter.level_y[0] && !hdr->loopfilter.level_y[1]) { - memset(lflvl_values, 0, 8 * 4 * 2 * n_seg); + memset(lflvl_values, 0, sizeof(*lflvl_values) * n_seg); return; } diff --git a/third_party/dav1d/src/meson.build b/third_party/dav1d/src/meson.build index dc4be5fd6f..cd19b70c38 100644 --- a/third_party/dav1d/src/meson.build +++ b/third_party/dav1d/src/meson.build @@ -106,6 +106,7 @@ if is_asm_enabled 'arm/64/loopfilter.S', 'arm/64/looprestoration.S', 'arm/64/mc.S', + 'arm/64/mc_dotprod.S', ) endif diff --git a/third_party/dav1d/src/refmvs.c b/third_party/dav1d/src/refmvs.c index 200afebde7..1da024b630 100644 --- a/third_party/dav1d/src/refmvs.c +++ b/third_party/dav1d/src/refmvs.c @@ -817,7 +817,9 @@ int dav1d_refmvs_init_frame(refmvs_frame *const rf, if (r_stride != rf->r_stride || n_tile_rows != rf->n_tile_rows) { if (rf->r) dav1d_freep_aligned(&rf->r); const int uses_2pass = n_tile_threads > 1 && n_frame_threads > 1; - rf->r = dav1d_alloc_aligned(ALLOC_REFMVS, sizeof(*rf->r) * 35 * r_stride * n_tile_rows * (1 + uses_2pass), 64); + /* sizeof(refmvs_block) == 12 but it's accessed using 16-byte loads in asm, + * so add 4 bytes of padding to avoid buffer overreads. */ + rf->r = dav1d_alloc_aligned(ALLOC_REFMVS, sizeof(*rf->r) * 35 * r_stride * n_tile_rows * (1 + uses_2pass) + 4, 64); if (!rf->r) return DAV1D_ERR(ENOMEM); rf->r_stride = r_stride; } diff --git a/third_party/dav1d/src/riscv/itx.h b/third_party/dav1d/src/riscv/itx.h index d3f9a03a03..e11b138348 100644 --- a/third_party/dav1d/src/riscv/itx.h +++ b/third_party/dav1d/src/riscv/itx.h @@ -28,34 +28,6 @@ #include "src/cpu.h" #include "src/itx.h" -#define decl_itx2_fns(w, h, opt) \ -decl_itx_fn(BF(dav1d_inv_txfm_add_dct_dct_##w##x##h, opt)); \ -decl_itx_fn(BF(dav1d_inv_txfm_add_identity_identity_##w##x##h, opt)) - -#define decl_itx12_fns(w, h, opt) \ -decl_itx2_fns(w, h, opt); \ -decl_itx_fn(BF(dav1d_inv_txfm_add_dct_adst_##w##x##h, opt)); \ -decl_itx_fn(BF(dav1d_inv_txfm_add_dct_flipadst_##w##x##h, opt)); \ -decl_itx_fn(BF(dav1d_inv_txfm_add_dct_identity_##w##x##h, opt)); \ -decl_itx_fn(BF(dav1d_inv_txfm_add_adst_dct_##w##x##h, opt)); \ -decl_itx_fn(BF(dav1d_inv_txfm_add_adst_adst_##w##x##h, opt)); \ -decl_itx_fn(BF(dav1d_inv_txfm_add_adst_flipadst_##w##x##h, opt)); \ -decl_itx_fn(BF(dav1d_inv_txfm_add_flipadst_dct_##w##x##h, opt)); \ -decl_itx_fn(BF(dav1d_inv_txfm_add_flipadst_adst_##w##x##h, opt)); \ -decl_itx_fn(BF(dav1d_inv_txfm_add_flipadst_flipadst_##w##x##h, opt)); \ -decl_itx_fn(BF(dav1d_inv_txfm_add_identity_dct_##w##x##h, opt)) - -#define decl_itx16_fns(w, h, opt) \ -decl_itx12_fns(w, h, opt); \ -decl_itx_fn(BF(dav1d_inv_txfm_add_adst_identity_##w##x##h, opt)); \ -decl_itx_fn(BF(dav1d_inv_txfm_add_flipadst_identity_##w##x##h, opt)); \ -decl_itx_fn(BF(dav1d_inv_txfm_add_identity_adst_##w##x##h, opt)); \ -decl_itx_fn(BF(dav1d_inv_txfm_add_identity_flipadst_##w##x##h, opt)) - -#define decl_itx17_fns(w, h, opt) \ -decl_itx16_fns(w, h, opt); \ -decl_itx_fn(BF(dav1d_inv_txfm_add_wht_wht_##w##x##h, opt)) - #define decl_itx_fns(ext) \ decl_itx17_fns( 4, 4, ext); \ decl_itx16_fns( 4, 8, ext); \ @@ -70,41 +42,6 @@ decl_itx16_fns(16, 16, ext) decl_itx_fns(rvv); static ALWAYS_INLINE void itx_dsp_init_riscv(Dav1dInvTxfmDSPContext *const c, int const bpc) { -#define assign_itx_fn(pfx, w, h, type, type_enum, ext) \ - c->itxfm_add[pfx##TX_##w##X##h][type_enum] = \ - BF(dav1d_inv_txfm_add_##type##_##w##x##h, ext) - -#define assign_itx1_fn(pfx, w, h, ext) \ - assign_itx_fn(pfx, w, h, dct_dct, DCT_DCT, ext) - -#define assign_itx2_fn(pfx, w, h, ext) \ - assign_itx1_fn(pfx, w, h, ext); \ - assign_itx_fn(pfx, w, h, identity_identity, IDTX, ext) - -#define assign_itx12_fn(pfx, w, h, ext) \ - assign_itx2_fn(pfx, w, h, ext); \ - assign_itx_fn(pfx, w, h, dct_adst, ADST_DCT, ext); \ - assign_itx_fn(pfx, w, h, dct_flipadst, FLIPADST_DCT, ext); \ - assign_itx_fn(pfx, w, h, dct_identity, H_DCT, ext); \ - assign_itx_fn(pfx, w, h, adst_dct, DCT_ADST, ext); \ - assign_itx_fn(pfx, w, h, adst_adst, ADST_ADST, ext); \ - assign_itx_fn(pfx, w, h, adst_flipadst, FLIPADST_ADST, ext); \ - assign_itx_fn(pfx, w, h, flipadst_dct, DCT_FLIPADST, ext); \ - assign_itx_fn(pfx, w, h, flipadst_adst, ADST_FLIPADST, ext); \ - assign_itx_fn(pfx, w, h, flipadst_flipadst, FLIPADST_FLIPADST, ext); \ - assign_itx_fn(pfx, w, h, identity_dct, V_DCT, ext) - -#define assign_itx16_fn(pfx, w, h, ext) \ - assign_itx12_fn(pfx, w, h, ext); \ - assign_itx_fn(pfx, w, h, adst_identity, H_ADST, ext); \ - assign_itx_fn(pfx, w, h, flipadst_identity, H_FLIPADST, ext); \ - assign_itx_fn(pfx, w, h, identity_adst, V_ADST, ext); \ - assign_itx_fn(pfx, w, h, identity_flipadst, V_FLIPADST, ext) - -#define assign_itx17_fn(pfx, w, h, ext) \ - assign_itx16_fn(pfx, w, h, ext); \ - assign_itx_fn(pfx, w, h, wht_wht, WHT_WHT, ext) - const unsigned flags = dav1d_get_cpu_flags(); if (!(flags & DAV1D_RISCV_CPU_FLAG_V)) return; diff --git a/third_party/dav1d/src/x86/ipred_avx2.asm b/third_party/dav1d/src/x86/ipred_avx2.asm index 35738e7c0b..2956ffaf29 100644 --- a/third_party/dav1d/src/x86/ipred_avx2.asm +++ b/third_party/dav1d/src/x86/ipred_avx2.asm @@ -66,7 +66,8 @@ z_filter_wh: db 7, 7, 11, 11, 15, 15, 19, 19, 19, 23, 23, 23, 31, 31, 31, 39 z_filter_k: db 0, 16, 0, 16, 0, 20, 0, 20, 8, 16, 8, 16 db 32, 16, 32, 16, 24, 20, 24, 20, 16, 16, 16, 16 db 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 8, 0 -z_filter_s: db 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7 +const \ +z_filter_s, db 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7 db 7, 8, 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, 14, 14, 15 db 15, 15, 15, 15, 15, 15, 15, 15 ; should be in one cache line pb_128: times 4 db 128 ; those are just placed here for alignment. diff --git a/third_party/dav1d/src/x86/itx.h b/third_party/dav1d/src/x86/itx.h index 346fde7d90..23d7a73806 100644 --- a/third_party/dav1d/src/x86/itx.h +++ b/third_party/dav1d/src/x86/itx.h @@ -30,34 +30,6 @@ #define BF_BPC(x, bits, suffix) x##_##bits##bpc_##suffix -#define decl_itx2_fns(w, h, opt) \ -decl_itx_fn(BF(dav1d_inv_txfm_add_dct_dct_##w##x##h, opt)); \ -decl_itx_fn(BF(dav1d_inv_txfm_add_identity_identity_##w##x##h, opt)) - -#define decl_itx12_fns(w, h, opt) \ -decl_itx2_fns(w, h, opt); \ -decl_itx_fn(BF(dav1d_inv_txfm_add_dct_adst_##w##x##h, opt)); \ -decl_itx_fn(BF(dav1d_inv_txfm_add_dct_flipadst_##w##x##h, opt)); \ -decl_itx_fn(BF(dav1d_inv_txfm_add_dct_identity_##w##x##h, opt)); \ -decl_itx_fn(BF(dav1d_inv_txfm_add_adst_dct_##w##x##h, opt)); \ -decl_itx_fn(BF(dav1d_inv_txfm_add_adst_adst_##w##x##h, opt)); \ -decl_itx_fn(BF(dav1d_inv_txfm_add_adst_flipadst_##w##x##h, opt)); \ -decl_itx_fn(BF(dav1d_inv_txfm_add_flipadst_dct_##w##x##h, opt)); \ -decl_itx_fn(BF(dav1d_inv_txfm_add_flipadst_adst_##w##x##h, opt)); \ -decl_itx_fn(BF(dav1d_inv_txfm_add_flipadst_flipadst_##w##x##h, opt)); \ -decl_itx_fn(BF(dav1d_inv_txfm_add_identity_dct_##w##x##h, opt)) - -#define decl_itx16_fns(w, h, opt) \ -decl_itx12_fns(w, h, opt); \ -decl_itx_fn(BF(dav1d_inv_txfm_add_adst_identity_##w##x##h, opt)); \ -decl_itx_fn(BF(dav1d_inv_txfm_add_flipadst_identity_##w##x##h, opt)); \ -decl_itx_fn(BF(dav1d_inv_txfm_add_identity_adst_##w##x##h, opt)); \ -decl_itx_fn(BF(dav1d_inv_txfm_add_identity_flipadst_##w##x##h, opt)) - -#define decl_itx17_fns(w, h, opt) \ -decl_itx16_fns(w, h, opt); \ -decl_itx_fn(BF(dav1d_inv_txfm_add_wht_wht_##w##x##h, opt)) - #define decl_itx_fns(ext) \ decl_itx17_fns( 4, 4, ext); \ decl_itx16_fns( 4, 8, ext); \ @@ -136,42 +108,6 @@ decl_itx_fn(dav1d_inv_txfm_add_wht_wht_4x4_16bpc_avx2); decl_itx_fn(BF(dav1d_inv_txfm_add_wht_wht_4x4, sse2)); static ALWAYS_INLINE void itx_dsp_init_x86(Dav1dInvTxfmDSPContext *const c, const int bpc) { -#define assign_itx_fn(pfx, w, h, type, type_enum, ext) \ - c->itxfm_add[pfx##TX_##w##X##h][type_enum] = \ - BF(dav1d_inv_txfm_add_##type##_##w##x##h, ext) - -#define assign_itx1_fn(pfx, w, h, ext) \ - assign_itx_fn(pfx, w, h, dct_dct, DCT_DCT, ext) - -#define assign_itx2_fn(pfx, w, h, ext) \ - assign_itx1_fn(pfx, w, h, ext); \ - assign_itx_fn(pfx, w, h, identity_identity, IDTX, ext) - -#define assign_itx12_fn(pfx, w, h, ext) \ - assign_itx2_fn(pfx, w, h, ext); \ - assign_itx_fn(pfx, w, h, dct_adst, ADST_DCT, ext); \ - assign_itx_fn(pfx, w, h, dct_flipadst, FLIPADST_DCT, ext); \ - assign_itx_fn(pfx, w, h, dct_identity, H_DCT, ext); \ - assign_itx_fn(pfx, w, h, adst_dct, DCT_ADST, ext); \ - assign_itx_fn(pfx, w, h, adst_adst, ADST_ADST, ext); \ - assign_itx_fn(pfx, w, h, adst_flipadst, FLIPADST_ADST, ext); \ - assign_itx_fn(pfx, w, h, flipadst_dct, DCT_FLIPADST, ext); \ - assign_itx_fn(pfx, w, h, flipadst_adst, ADST_FLIPADST, ext); \ - assign_itx_fn(pfx, w, h, flipadst_flipadst, FLIPADST_FLIPADST, ext); \ - assign_itx_fn(pfx, w, h, identity_dct, V_DCT, ext) - -#define assign_itx16_fn(pfx, w, h, ext) \ - assign_itx12_fn(pfx, w, h, ext); \ - assign_itx_fn(pfx, w, h, adst_identity, H_ADST, ext); \ - assign_itx_fn(pfx, w, h, flipadst_identity, H_FLIPADST, ext); \ - assign_itx_fn(pfx, w, h, identity_adst, V_ADST, ext); \ - assign_itx_fn(pfx, w, h, identity_flipadst, V_FLIPADST, ext) - -#define assign_itx17_fn(pfx, w, h, ext) \ - assign_itx16_fn(pfx, w, h, ext); \ - assign_itx_fn(pfx, w, h, wht_wht, WHT_WHT, ext) - - #define assign_itx_bpc_fn(pfx, w, h, type, type_enum, bpc, ext) \ c->itxfm_add[pfx##TX_##w##X##h][type_enum] = \ BF_BPC(dav1d_inv_txfm_add_##type##_##w##x##h, bpc, ext) diff --git a/third_party/dav1d/src/x86/mc16_avx2.asm b/third_party/dav1d/src/x86/mc16_avx2.asm index 42e2a5525e..6b4424946b 100644 --- a/third_party/dav1d/src/x86/mc16_avx2.asm +++ b/third_party/dav1d/src/x86/mc16_avx2.asm @@ -1222,7 +1222,7 @@ cglobal prep_bilin_16bpc, 3, 7, 0, tmp, src, stride, w, h, mxy, stride3 %assign FILTER_SMOOTH (1*15 << 16) | 4*15 %assign FILTER_SHARP (2*15 << 16) | 3*15 -%macro FN 4 ; prefix, type, type_h, type_v +%macro FN 4-5 ; prefix, type, type_h, type_v, jmp_to cglobal %1_%2_16bpc mov t0d, FILTER_%3 %ifidn %3, %4 @@ -1230,8 +1230,8 @@ cglobal %1_%2_16bpc %else mov t1d, FILTER_%4 %endif -%ifnidn %2, regular ; skip the jump in the last filter - jmp mangle(private_prefix %+ _%1_16bpc %+ SUFFIX) +%if %0 == 5 ; skip the jump in the last filter + jmp mangle(private_prefix %+ _%5 %+ SUFFIX) %endif %endmacro @@ -1242,22 +1242,17 @@ DECLARE_REG_TMP 7, 8 %endif %define PUT_8TAP_FN FN put_8tap, -PUT_8TAP_FN sharp, SHARP, SHARP -PUT_8TAP_FN sharp_smooth, SHARP, SMOOTH -PUT_8TAP_FN smooth_sharp, SMOOTH, SHARP -PUT_8TAP_FN smooth, SMOOTH, SMOOTH -PUT_8TAP_FN sharp_regular, SHARP, REGULAR -PUT_8TAP_FN regular_sharp, REGULAR, SHARP -PUT_8TAP_FN smooth_regular, SMOOTH, REGULAR -PUT_8TAP_FN regular_smooth, REGULAR, SMOOTH +PUT_8TAP_FN smooth, SMOOTH, SMOOTH, put_6tap_16bpc +PUT_8TAP_FN smooth_regular, SMOOTH, REGULAR, put_6tap_16bpc +PUT_8TAP_FN regular_smooth, REGULAR, SMOOTH, put_6tap_16bpc PUT_8TAP_FN regular, REGULAR, REGULAR -cglobal put_8tap_16bpc, 4, 9, 0, dst, ds, src, ss, w, h, mx, my +cglobal put_6tap_16bpc, 4, 9, 0, dst, ds, src, ss, w, h, mx, my %define base r8-put_avx2 imul mxd, mxm, 0x010101 - add mxd, t0d ; 8tap_h, mx, 4tap_h + add mxd, t0d ; 6tap_h, mx, 4tap_h imul myd, mym, 0x010101 - add myd, t1d ; 8tap_v, my, 4tap_v + add myd, t1d ; 6tap_v, my, 4tap_v lea r8, [put_avx2] movifnidn wd, wm movifnidn hd, hm @@ -1265,6 +1260,7 @@ cglobal put_8tap_16bpc, 4, 9, 0, dst, ds, src, ss, w, h, mx, my jnz .h test myd, 0xf00 jnz .v +.put: tzcnt wd, wd movzx wd, word [r8+wq*2+table_offset(put,)] add wq, r8 @@ -1337,43 +1333,36 @@ cglobal put_8tap_16bpc, 4, 9, 0, dst, ds, src, ss, w, h, mx, my cmp wd, 4 je .h_w4 jl .h_w2 - WIN64_SPILL_XMM 13 + WIN64_SPILL_XMM 11 shr mxd, 16 - sub srcq, 6 - vpbroadcastq m0, [base+subpel_filters+mxq*8] - vbroadcasti128 m6, [subpel_h_shufA] - vbroadcasti128 m7, [subpel_h_shufB] + sub srcq, 4 + vpbroadcastq m0, [base+subpel_filters+1+mxq*8] + vbroadcasti128 m6, [base+subpel_h_shufA] punpcklbw m0, m0 psraw m0, 8 ; sign-extend - pshufd m8, m0, q0000 - pshufd m9, m0, q1111 - pshufd m10, m0, q2222 - pshufd m11, m0, q3333 - cmp wd, 8 - jg .h_w16 + pshufd m7, m0, q0000 + pshufd m8, m0, q1111 + pshufd m9, m0, q2222 + sub wd, 16 + jge .h_w16 .h_w8: -%macro PUT_8TAP_H 5 ; dst/src+0, src+8, src+16, tmp[1-2] - pshufb m%4, m%1, m7 ; 2 3 3 4 4 5 5 6 - pshufb m%1, m6 ; 0 1 1 2 2 3 3 4 - pmaddwd m%5, m9, m%4 ; abcd1 - pmaddwd m%1, m8 ; abcd0 - pshufb m%2, m7 ; 6 7 7 8 8 9 9 a - shufpd m%4, m%2, 0x05 ; 4 5 5 6 6 7 7 8 - paddd m%5, m4 - paddd m%1, m%5 - pmaddwd m%5, m11, m%2 ; abcd3 - paddd m%1, m%5 - pmaddwd m%5, m10, m%4 ; abcd2 - pshufb m%3, m7 ; a b b c c d d e - pmaddwd m%4, m8 ; efgh0 - paddd m%1, m%5 - pmaddwd m%5, m9, m%2 ; efgh1 - shufpd m%2, m%3, 0x05 ; 8 9 9 a a b b c - pmaddwd m%3, m11 ; efgh3 - pmaddwd m%2, m10 ; efgh2 +%macro PUT_6TAP_H 5 ; dst/src+0, src+8, src+16, tmp[1-2] + pshufb m%1, m6 ; 01 12 23 34 + pshufb m%2, m6 ; 45 56 67 78 + pmaddwd m%4, m7, m%1 ; a0 + pshufb m%3, m6 ; 89 9a ab bc + pmaddwd m%5, m9, m%2 ; a2 + shufpd m%1, m%2, 0x05 ; 23 34 45 56 + paddd m%4, m%5 ; a0+a2 + pmaddwd m%5, m7, m%2 ; b0 + shufpd m%2, m%3, 0x05 ; 67 78 89 9a + pmaddwd m%3, m9 ; b2 + pmaddwd m%1, m8 ; a1 + pmaddwd m%2, m8 ; b1 + paddd m%3, m%5 ; b0+b2 paddd m%4, m4 - paddd m%4, m%5 - paddd m%3, m%4 + paddd m%3, m4 + paddd m%1, m%4 paddd m%2, m%3 psrad m%1, 6 psrad m%2, 6 @@ -1384,9 +1373,9 @@ cglobal put_8tap_16bpc, 4, 9, 0, dst, ds, src, ss, w, h, mx, my vinserti128 m0, [srcq+ssq*1+ 0], 1 movu xm2, [srcq+ssq*0+16] vinserti128 m2, [srcq+ssq*1+16], 1 - lea srcq, [srcq+ssq*2] shufpd m1, m0, m2, 0x05 - PUT_8TAP_H 0, 1, 2, 3, 12 + lea srcq, [srcq+ssq*2] + PUT_6TAP_H 0, 1, 2, 3, 10 mova [dstq+dsq*0], xm0 vextracti128 [dstq+dsq*1], m0, 1 lea dstq, [dstq+dsq*2] @@ -1396,13 +1385,13 @@ cglobal put_8tap_16bpc, 4, 9, 0, dst, ds, src, ss, w, h, mx, my .h_w16: mov r6d, wd .h_w16_loop: - movu m0, [srcq+r6*2-32] - movu m1, [srcq+r6*2-24] - movu m2, [srcq+r6*2-16] - PUT_8TAP_H 0, 1, 2, 3, 12 - mova [dstq+r6*2-32], m0 + movu m0, [srcq+r6*2+ 0] + movu m1, [srcq+r6*2+ 8] + movu m2, [srcq+r6*2+16] + PUT_6TAP_H 0, 1, 2, 3, 10 + mova [dstq+r6*2], m0 sub r6d, 16 - jg .h_w16_loop + jge .h_w16_loop add srcq, ssq add dstq, dsq dec hd @@ -1411,10 +1400,449 @@ cglobal put_8tap_16bpc, 4, 9, 0, dst, ds, src, ss, w, h, mx, my .v: movzx mxd, myb shr myd, 16 - cmp hd, 4 - cmovle myd, mxd + cmp hd, 6 + cmovs myd, mxd + vpbroadcastq m0, [base+subpel_filters+1+myq*8] + WIN64_SPILL_XMM 10, 12 + vpbroadcastd m5, [pd_32] + vpbroadcastw m6, r8m + punpcklbw m0, m0 + mov r6, ssq + psraw m0, 8 ; sign-extend + neg r6 + pshufd m7, m0, q0000 + pshufd m8, m0, q1111 + pshufd m9, m0, q2222 + cmp wd, 4 + jg .v_w8 + je .v_w4 +.v_w2: + movd xm2, [srcq+r6 *2] + pinsrd xm2, [srcq+r6 *1], 1 + pinsrd xm2, [srcq+ssq*0], 2 + pinsrd xm2, [srcq+ssq*1], 3 ; 0 1 2 3 + lea srcq, [srcq+ssq*2] + movd xm0, [srcq+ssq*0] + palignr xm3, xm0, xm2, 4 ; 1 2 3 4 + punpcklwd xm1, xm2, xm3 ; 01 12 + punpckhwd xm2, xm3 ; 23 34 +.v_w2_loop: + movd xm3, [srcq+ssq*1] + pmaddwd xm4, xm7, xm1 ; a0 b0 + mova xm1, xm2 + pmaddwd xm2, xm8 ; a1 b1 + lea srcq, [srcq+ssq*2] + paddd xm4, xm2 + punpckldq xm2, xm0, xm3 ; 4 5 + movd xm0, [srcq+ssq*0] + punpckldq xm3, xm0 ; 5 6 + punpcklwd xm2, xm3 ; 45 56 + pmaddwd xm3, xm9, xm2 ; a2 b2 + paddd xm4, xm5 + paddd xm4, xm3 + psrad xm4, 6 + packusdw xm4, xm4 + pminsw xm4, xm6 + movd [dstq+dsq*0], xm4 + pextrd [dstq+dsq*1], xm4, 1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .v_w2_loop + RET +.v_w4: + movq xm1, [srcq+r6 *2] + vpbroadcastq m3, [srcq+r6 *1] + vpbroadcastq m2, [srcq+ssq*0] + vpbroadcastq m4, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + vpbroadcastq m0, [srcq+ssq*0] + vpblendd m1, m3, 0x30 + vpblendd m3, m2, 0x30 + punpcklwd m1, m3 ; 01 12 + vpblendd m2, m4, 0x30 + vpblendd m4, m0, 0x30 + punpcklwd m2, m4 ; 23 34 +.v_w4_loop: + vpbroadcastq m3, [srcq+ssq*1] + pmaddwd m4, m7, m1 ; a0 b0 + mova m1, m2 + pmaddwd m2, m8 ; a1 b1 + lea srcq, [srcq+ssq*2] + paddd m4, m2 + vpblendd m2, m0, m3, 0x30 + vpbroadcastq m0, [srcq+ssq*0] + vpblendd m3, m0, 0x30 + punpcklwd m2, m3 ; 45 56 + pmaddwd m3, m9, m2 ; a2 b2 + paddd m4, m5 + paddd m4, m3 + psrad m4, 6 + vextracti128 xm3, m4, 1 + packusdw xm4, xm3 + pminsw xm4, xm6 + movq [dstq+dsq*0], xm4 + movhps [dstq+dsq*1], xm4 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .v_w4_loop + RET +.v_w8: + shl wd, 5 + WIN64_PUSH_XMM 12 + lea wd, [hq+wq-256] +.v_w8_loop0: + vbroadcasti128 m3, [srcq+r6 *2] + vbroadcasti128 m4, [srcq+r6 *1] + lea r7, [srcq+ssq*2] + vbroadcasti128 m0, [srcq+ssq*0] + vbroadcasti128 m1, [srcq+ssq*1] + mov r8, dstq + vbroadcasti128 m2, [r7+ssq*0] + shufpd m3, m0, 0x0c + shufpd m4, m1, 0x0c + punpcklwd m1, m3, m4 ; 01 + punpckhwd m3, m4 ; 23 + shufpd m0, m2, 0x0c + punpcklwd m2, m4, m0 ; 12 + punpckhwd m4, m0 ; 34 +.v_w8_loop: + vbroadcasti128 m5, [r7+ssq*1] + pmaddwd m10, m7, m1 ; a0 + lea r7, [r7+ssq*2] + pmaddwd m11, m7, m2 ; b0 + mova m1, m3 + pmaddwd m3, m8 ; a1 + mova m2, m4 + pmaddwd m4, m8 ; b1 + paddd m10, m3 + vbroadcasti128 m3, [r7+ssq*0] + paddd m11, m4 + shufpd m4, m0, m5, 0x0d + shufpd m0, m5, m3, 0x0c + punpcklwd m3, m4, m0 ; 45 + punpckhwd m4, m0 ; 56 + pmaddwd m5, m9, m3 ; a2 + paddd m10, m5 + pmaddwd m5, m9, m4 ; b2 + paddd m5, m11 + psrad m10, 5 + psrad m5, 5 + packusdw m10, m5 + pxor m5, m5 + pavgw m5, m10 + pminsw m5, m6 + vpermq m5, m5, q3120 + mova [r8+dsq*0], xm5 + vextracti128 [r8+dsq*1], m5, 1 + lea r8, [r8+dsq*2] + sub hd, 2 + jg .v_w8_loop + add srcq, 16 + add dstq, 16 + movzx hd, wb + sub wd, 1<<8 + jg .v_w8_loop0 + RET +.hv: + WIN64_SPILL_XMM 12, 16 + vpbroadcastd m10, [pd_512] + vpbroadcastw m11, r8m + cmp wd, 4 + jg .hv_w8 + movzx mxd, mxb + vpbroadcastd m0, [base+subpel_filters+mxq*8+2] + movzx mxd, myb + shr myd, 16 + cmp hd, 6 + cmovs myd, mxd + vpbroadcastq m1, [base+subpel_filters+1+myq*8] + mov r6, ssq + sub srcq, 2 + neg r6 + pxor m6, m6 + punpcklbw m6, m0 + punpcklbw m1, m1 + psraw m1, 8 ; sign-extend + test dword r8m, 0x800 + jz .hv_10bit + psraw m6, 2 + psllw m1, 2 +.hv_10bit: + pshufd m7, m1, q0000 + pshufd m8, m1, q1111 + pshufd m9, m1, q2222 + cmp wd, 4 + je .hv_w4 + vbroadcasti128 m5, [subpel_h_shuf2] + vbroadcasti128 m0, [srcq+ssq*0] + vinserti128 m2, m0, [srcq+r6*2], 1 ; 2 0 + movu xm1, [srcq+ssq*1] + vinserti128 m1, [srcq+r6 *1], 1 ; 3 1 + lea srcq, [srcq+ssq*2] + vinserti128 m0, [srcq+ssq*0], 0 ; 4 2 + REPX {pshufb x, m5}, m2, m1, m0 + REPX {pmaddwd x, m6}, m2, m1, m0 + phaddd m2, m1 + phaddd m1, m0 + paddd m2, m10 + paddd m1, m10 + psrad m2, 10 + psrad m1, 10 + packssdw m2, m1 ; 2 3 3 4 0 1 1 2 + punpckhqdq m0, m2, m2 + punpcklwd m2, m0 ; 23 34 + vextracti128 xm1, m2, 1 ; 01 12 +.hv_w2_loop: + movu xm3, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + movu xm4, [srcq+ssq*0] + pshufb xm3, xm5 + pshufb xm4, xm5 + pmaddwd xm3, xm6 + pmaddwd xm4, xm6 + phaddd xm3, xm4 + pmaddwd xm4, xm7, xm1 ; a0 b0 + mova xm1, xm2 + pmaddwd xm2, xm8 ; a1 b1 + paddd xm4, xm2 + paddd xm3, xm10 + psrad xm3, 10 + packssdw xm3, xm3 + palignr xm2, xm3, xm0, 12 + mova xm0, xm3 + punpcklwd xm2, xm0 ; 45 56 + pmaddwd xm3, xm9, xm2 ; a2 b2 + paddd xm4, xm10 + paddd xm4, xm3 + psrad xm4, 10 + packusdw xm4, xm4 + pminsw xm4, xm11 + movd [dstq+dsq*0], xm4 + pextrd [dstq+dsq*1], xm4, 1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .hv_w2_loop + RET +.hv_w4: + WIN64_PUSH_XMM 14 + vbroadcasti128 m12, [subpel_h_shufA] + pshufd m5, m6, q0000 + vbroadcasti128 m13, [subpel_h_shufB] + pshufd m6, m6, q1111 + movu xm2, [srcq+r6 *2] + vinserti128 m2, [srcq+r6 *1], 1 ; 0 1 + movu xm0, [srcq+ssq*0] + vinserti128 m0, [srcq+ssq*1], 1 ; 2 3 + lea srcq, [srcq+ssq*2] + movu xm3, [srcq+ssq*0] ; 4 + pshufb m1, m2, m12 + pmaddwd m1, m5 + pshufb m2, m13 + pmaddwd m2, m6 + pshufb m4, m0, m12 + pmaddwd m4, m5 + pshufb m0, m13 + pmaddwd m0, m6 + paddd m2, m1 + pshufb xm1, xm3, xm12 + pmaddwd xm1, xm5 + pshufb xm3, xm13 + pmaddwd xm3, xm6 + paddd m0, m4 + paddd m2, m10 + paddd xm1, xm10 + paddd m0, m10 + paddd xm3, xm1 + REPX {psrad x, 10}, m2, m0, xm3 + packssdw m2, m0 ; 0 2 1 3 + packssdw xm0, xm3 ; 2 4 + vperm2i128 m0, m2, 0x03 + punpcklwd m1, m2, m0 ; 01 12 + punpckhwd m2, m0 ; 23 34 +.hv_w4_loop: + movu xm3, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + vinserti128 m3, [srcq+ssq*0], 1 + pmaddwd m4, m7, m1 ; a0 b0 + mova m1, m2 + pmaddwd m2, m8 ; a1 b1 + paddd m4, m2 + pshufb m2, m3, m12 + pmaddwd m2, m5 + pshufb m3, m13 + pmaddwd m3, m6 + paddd m2, m10 + paddd m3, m2 + psrad m3, 10 + packssdw m3, m3 ; 5 5 6 6 + vperm2i128 m2, m0, m3, 0x21 + mova m0, m3 + punpckhwd m2, m3 ; 45 56 + pmaddwd m3, m9, m2 ; a2 b2 + paddd m4, m10 + paddd m4, m3 + psrad m4, 10 + vextracti128 xm3, m4, 1 + packusdw xm4, xm3 + pminsw xm4, xm11 + movq [dstq+dsq*0], xm4 + movhps [dstq+dsq*1], xm4 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .hv_w4_loop + RET +.hv_w8: + WIN64_PUSH_XMM 16, 12 + shr mxd, 16 + vbroadcasti128 m12, [subpel_h_shufA] + vpbroadcastq m2, [base+subpel_filters+1+mxq*8] + movzx mxd, myb + shr myd, 16 + cmp hd, 6 + cmovs myd, mxd + pmovsxbw xm1, [base+subpel_filters+1+myq*8] + shl wd, 5 + mov r6, ssq + sub srcq, 4 + pxor m0, m0 + neg r6 + punpcklbw m0, m2 + lea wd, [hq+wq-256] + test dword r8m, 0x800 + jz .hv_w8_10bit + psraw m0, 2 + psllw xm1, 2 +.hv_w8_10bit: + pshufd m7, m0, q0000 + pshufd m8, m0, q1111 +%if WIN64 + %define v_mul (rsp+stack_offset+40) ; r4m +%else + %define v_mul (rsp+stack_offset+ 8) ; r6m +%endif + mova [v_mul], xm1 + pshufd m9, m0, q2222 +.hv_w8_loop0: + vbroadcasti128 m0, [srcq+ssq*0+ 0] + vinserti128 m3, m0, [srcq+r6*2+ 0], 0 + lea r7, [srcq+ssq*2] + vbroadcasti128 m2, [srcq+ssq*0+16] + vinserti128 m1, m2, [srcq+r6*2+16], 0 + mov r8, dstq + vinserti128 m0, [r7 +ssq*0+ 0], 1 + vinserti128 m2, [r7 +ssq*0+16], 1 + shufpd m4, m3, m1, 0x05 +%macro PUT_6TAP_HV_H 5 ; dst/src+0, src+8, src+16, tmp[1-2] + pshufb m%1, m12 ; 01 12 23 34 + pshufb m%2, m12 ; 45 56 67 78 + pmaddwd m%4, m7, m%1 ; a0 + pshufb m%3, m12 ; 89 9a ab bc + pmaddwd m%5, m9, m%2 ; a2 + shufpd m%1, m%2, 0x05 ; 23 34 45 56 + paddd m%4, m%5 ; a0+a2 + pmaddwd m%5, m7, m%2 ; b0 + shufpd m%2, m%3, 0x05 ; 67 78 89 9a + pmaddwd m%3, m9 ; b2 + pmaddwd m%1, m8 ; a1 + pmaddwd m%2, m8 ; b1 + paddd m%3, m%5 ; b0+b2 + paddd m%4, m10 + paddd m%3, m10 + paddd m%1, m%4 + paddd m%2, m%3 + psrad m%1, 10 + psrad m%2, 10 + packssdw m%1, m%2 +%endmacro + PUT_6TAP_HV_H 3, 4, 1, 5, 6 ; 0 2 + movu xm4, [srcq+r6 *1+ 0] + vinserti128 m4, [srcq+ssq*1+ 0], 1 + shufpd m1, m0, m2, 0x05 + PUT_6TAP_HV_H 0, 1, 2, 5, 6 ; 2 4 + movu xm2, [srcq+r6 *1+16] + vinserti128 m2, [srcq+ssq*1+16], 1 + shufpd m1, m4, m2, 0x05 + PUT_6TAP_HV_H 4, 1, 2, 5, 6 ; 1 3 + vpermq m3, m3, q3120 + vpermq m4, m4, q3120 + vpermq m0, m0, q3120 + punpcklwd m1, m3, m4 ; 01 + punpckhwd m3, m4 ; 23 + punpcklwd m2, m4, m0 ; 12 + punpckhwd m4, m0 ; 34 +.hv_w8_loop: + vpbroadcastd m15, [v_mul+4*0] + vpbroadcastd m13, [v_mul+4*1] + movu xm5, [r7+ssq*1+ 0] + movu xm6, [r7+ssq*1+16] + lea r7, [r7+ssq*2] + pmaddwd m14, m15, m1 ; a0 + pmaddwd m15, m2 ; b0 + vinserti128 m5, [r7+ssq*0+ 0], 1 + vinserti128 m6, [r7+ssq*0+16], 1 + mova m1, m3 + pmaddwd m3, m13 ; a1 + mova m2, m4 + pmaddwd m4, m13 ; b1 + paddd m14, m3 + shufpd m3, m5, m6, 0x05 + paddd m15, m4 + PUT_6TAP_HV_H 5, 3, 6, 4, 13 ; 5 6 + vpbroadcastd m6, [v_mul+4*2] + vpermq m5, m5, q3120 + shufpd m4, m0, m5, 0x05 + mova m0, m5 + punpcklwd m3, m4, m5 ; 45 + punpckhwd m4, m5 ; 56 + pmaddwd m5, m6, m3 ; a2 + pmaddwd m6, m4 ; b2 + paddd m14, m10 + paddd m15, m10 + paddd m5, m14 + paddd m6, m15 + psrad m5, 10 + psrad m6, 10 + packusdw m5, m6 + pminsw m5, m11 + vpermq m5, m5, q3120 + mova [r8+dsq*0], xm5 + vextracti128 [r8+dsq*1], m5, 1 + lea r8, [r8+dsq*2] + sub hd, 2 + jg .hv_w8_loop + add srcq, 16 + add dstq, 16 + movzx hd, wb + sub wd, 1<<8 + jg .hv_w8_loop0 + RET + +PUT_8TAP_FN smooth_sharp, SMOOTH, SHARP, put_8tap_16bpc +PUT_8TAP_FN sharp_smooth, SHARP, SMOOTH, put_8tap_16bpc +PUT_8TAP_FN regular_sharp, REGULAR, SHARP, put_8tap_16bpc +PUT_8TAP_FN sharp_regular, SHARP, REGULAR, put_8tap_16bpc +PUT_8TAP_FN sharp, SHARP, SHARP + +cglobal put_8tap_16bpc, 4, 9, 0, dst, ds, src, ss, w, h, mx, my +%define base r8-put_avx2 + imul mxd, mxm, 0x010101 + add mxd, t0d ; 8tap_h, mx, 4tap_h + imul myd, mym, 0x010101 + add myd, t1d ; 8tap_v, my, 4tap_v + lea r8, [put_avx2] + movifnidn wd, wm + movifnidn hd, hm + test mxd, 0xf00 + jnz .h + test myd, 0xf00 + jz mangle(private_prefix %+ _put_6tap_16bpc_avx2).put +.v: + movzx mxd, myb + shr myd, 16 + cmp hd, 6 + cmovs myd, mxd vpbroadcastq m0, [base+subpel_filters+myq*8] - WIN64_SPILL_XMM 15 + WIN64_SPILL_XMM 12, 15 vpbroadcastd m6, [pd_32] vpbroadcastw m7, r8m lea r6, [ssq*3] @@ -1518,19 +1946,19 @@ cglobal put_8tap_16bpc, 4, 9, 0, dst, ds, src, ss, w, h, mx, my RET .v_w8: shl wd, 5 - mov r7, srcq - mov r8, dstq + WIN64_PUSH_XMM 15 lea wd, [hq+wq-256] .v_w8_loop0: vbroadcasti128 m4, [srcq+ssq*0] vbroadcasti128 m5, [srcq+ssq*1] + lea r7, [srcq+ssq*4] vbroadcasti128 m0, [srcq+r6 ] vbroadcasti128 m6, [srcq+ssq*2] - lea srcq, [srcq+ssq*4] - vbroadcasti128 m1, [srcq+ssq*0] - vbroadcasti128 m2, [srcq+ssq*1] - vbroadcasti128 m3, [srcq+ssq*2] - add srcq, r6 + mov r8, dstq + vbroadcasti128 m1, [r7+ssq*0] + vbroadcasti128 m2, [r7+ssq*1] + vbroadcasti128 m3, [r7+ssq*2] + add r7, r6 shufpd m4, m0, 0x0c shufpd m5, m1, 0x0c punpcklwd m1, m4, m5 ; 01 @@ -1542,7 +1970,7 @@ cglobal put_8tap_16bpc, 4, 9, 0, dst, ds, src, ss, w, h, mx, my punpcklwd m3, m6, m0 ; 23 punpckhwd m6, m0 ; 56 .v_w8_loop: - vbroadcasti128 m14, [srcq+ssq*0] + vbroadcasti128 m14, [r7+ssq*0] pmaddwd m12, m8, m1 ; a0 pmaddwd m13, m8, m2 ; b0 mova m1, m3 @@ -1556,8 +1984,8 @@ cglobal put_8tap_16bpc, 4, 9, 0, dst, ds, src, ss, w, h, mx, my pmaddwd m5, m10 ; a2 pmaddwd m6, m10 ; b2 paddd m12, m5 - vbroadcasti128 m5, [srcq+ssq*1] - lea srcq, [srcq+ssq*2] + vbroadcasti128 m5, [r7+ssq*1] + lea r7, [r7+ssq*2] paddd m13, m6 shufpd m6, m0, m14, 0x0d shufpd m0, m14, m5, 0x0c @@ -1574,19 +2002,99 @@ cglobal put_8tap_16bpc, 4, 9, 0, dst, ds, src, ss, w, h, mx, my pavgw m12, m13 pminsw m12, m7 vpermq m12, m12, q3120 - mova [dstq+dsq*0], xm12 - vextracti128 [dstq+dsq*1], m12, 1 - lea dstq, [dstq+dsq*2] + mova [r8+dsq*0], xm12 + vextracti128 [r8+dsq*1], m12, 1 + lea r8, [r8+dsq*2] sub hd, 2 jg .v_w8_loop - add r7, 16 - add r8, 16 + add srcq, 16 + add dstq, 16 movzx hd, wb - mov srcq, r7 - mov dstq, r8 sub wd, 1<<8 jg .v_w8_loop0 RET +.h: + RESET_STACK_STATE + test myd, 0xf00 + jnz .hv + mov r7d, r8m + vpbroadcastw m5, r8m + shr r7d, 11 + vpbroadcastd m4, [base+put_8tap_h_rnd+r7*4] + cmp wd, 4 + jl mangle(private_prefix %+ _put_6tap_16bpc_avx2).h_w2 + je mangle(private_prefix %+ _put_6tap_16bpc_avx2).h_w4 + WIN64_SPILL_XMM 13 + shr mxd, 16 + sub srcq, 6 + vpbroadcastq m0, [base+subpel_filters+mxq*8] + vbroadcasti128 m6, [subpel_h_shufA] + vbroadcasti128 m7, [subpel_h_shufB] + punpcklbw m0, m0 + psraw m0, 8 ; sign-extend + pshufd m8, m0, q0000 + pshufd m9, m0, q1111 + pshufd m10, m0, q2222 + pshufd m11, m0, q3333 + sub wd, 16 + jge .h_w16 +.h_w8: +%macro PUT_8TAP_H 5 ; dst/src+0, src+8, src+16, tmp[1-2] + pshufb m%4, m%1, m7 ; 2 3 3 4 4 5 5 6 + pshufb m%1, m6 ; 0 1 1 2 2 3 3 4 + pmaddwd m%5, m9, m%4 ; abcd1 + pmaddwd m%1, m8 ; abcd0 + pshufb m%2, m7 ; 6 7 7 8 8 9 9 a + shufpd m%4, m%2, 0x05 ; 4 5 5 6 6 7 7 8 + paddd m%5, m4 + paddd m%1, m%5 + pmaddwd m%5, m11, m%2 ; abcd3 + paddd m%1, m%5 + pmaddwd m%5, m10, m%4 ; abcd2 + pshufb m%3, m7 ; a b b c c d d e + pmaddwd m%4, m8 ; efgh0 + paddd m%1, m%5 + pmaddwd m%5, m9, m%2 ; efgh1 + shufpd m%2, m%3, 0x05 ; 8 9 9 a a b b c + pmaddwd m%3, m11 ; efgh3 + pmaddwd m%2, m10 ; efgh2 + paddd m%4, m4 + paddd m%4, m%5 + paddd m%3, m%4 + paddd m%2, m%3 + psrad m%1, 6 + psrad m%2, 6 + packusdw m%1, m%2 + pminsw m%1, m5 +%endmacro + movu xm0, [srcq+ssq*0+ 0] + vinserti128 m0, [srcq+ssq*1+ 0], 1 + movu xm2, [srcq+ssq*0+16] + vinserti128 m2, [srcq+ssq*1+16], 1 + lea srcq, [srcq+ssq*2] + shufpd m1, m0, m2, 0x05 + PUT_8TAP_H 0, 1, 2, 3, 12 + mova [dstq+dsq*0], xm0 + vextracti128 [dstq+dsq*1], m0, 1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .h_w8 + RET +.h_w16: + mov r6d, wd +.h_w16_loop: + movu m0, [srcq+r6*2+ 0] + movu m1, [srcq+r6*2+ 8] + movu m2, [srcq+r6*2+16] + PUT_8TAP_H 0, 1, 2, 3, 12 + mova [dstq+r6*2], m0 + sub r6d, 16 + jge .h_w16_loop + add srcq, ssq + add dstq, dsq + dec hd + jg .h_w16 + RET .hv: WIN64_SPILL_XMM 16 vpbroadcastw m15, r8m @@ -1596,8 +2104,8 @@ cglobal put_8tap_16bpc, 4, 9, 0, dst, ds, src, ss, w, h, mx, my vpbroadcastd m0, [base+subpel_filters+mxq*8+2] movzx mxd, myb shr myd, 16 - cmp hd, 4 - cmovle myd, mxd + cmp hd, 6 + cmovs myd, mxd vpbroadcastq m1, [base+subpel_filters+myq*8] vpbroadcastd m6, [pd_512] lea r6, [ssq*3] @@ -1773,17 +2281,15 @@ cglobal put_8tap_16bpc, 4, 9, 0, dst, ds, src, ss, w, h, mx, my vpbroadcastq m2, [base+subpel_filters+mxq*8] movzx mxd, myb shr myd, 16 - cmp hd, 4 - cmovle myd, mxd + cmp hd, 6 + cmovs myd, mxd pmovsxbw xm1, [base+subpel_filters+myq*8] shl wd, 5 lea r6, [ssq*3] sub srcq, 6 - sub srcq, r6 pxor m0, m0 + sub srcq, r6 punpcklbw m0, m2 - mov r7, srcq - mov r8, dstq lea wd, [hq+wq-256] test dword r8m, 0x800 jz .hv_w8_10bit @@ -1792,14 +2298,9 @@ cglobal put_8tap_16bpc, 4, 9, 0, dst, ds, src, ss, w, h, mx, my .hv_w8_10bit: pshufd m11, m0, q0000 pshufd m12, m0, q1111 + mova [v_mul], xm1 pshufd m13, m0, q2222 pshufd m14, m0, q3333 -%if WIN64 - %define v_mul (rsp+stack_offset+40) ; r4m -%else - %define v_mul (rsp-24) ; red zone -%endif - mova [v_mul], xm1 .hv_w8_loop0: %macro PUT_8TAP_HV_H 3 ; dst/src+0, src+8, src+16 pshufb m2, m%1, m9 ; 2 3 3 4 4 5 5 6 @@ -1830,14 +2331,16 @@ cglobal put_8tap_16bpc, 4, 9, 0, dst, ds, src, ss, w, h, mx, my %endmacro movu xm4, [srcq+r6 *1+ 0] vbroadcasti128 m8, [subpel_h_shufA] + lea r7, [srcq+ssq*4] movu xm6, [srcq+r6 *1+ 8] vbroadcasti128 m9, [subpel_h_shufB] + mov r8, dstq movu xm0, [srcq+r6 *1+16] vpbroadcastd m10, [pd_512] movu xm5, [srcq+ssq*0+ 0] - vinserti128 m5, [srcq+ssq*4+ 0], 1 + vinserti128 m5, [r7 +ssq*0+ 0], 1 movu xm1, [srcq+ssq*0+16] - vinserti128 m1, [srcq+ssq*4+16], 1 + vinserti128 m1, [r7 +ssq*0+16], 1 shufpd m7, m5, m1, 0x05 INIT_XMM avx2 PUT_8TAP_HV_H 4, 6, 0 ; 3 @@ -1851,10 +2354,9 @@ cglobal put_8tap_16bpc, 4, 9, 0, dst, ds, src, ss, w, h, mx, my PUT_8TAP_HV_H 0, 7, 1 ; 2 6 movu xm6, [srcq+ssq*1+ 0] movu xm1, [srcq+ssq*1+16] - lea srcq, [srcq+ssq*4] - vinserti128 m6, [srcq+ssq*1+ 0], 1 - vinserti128 m1, [srcq+ssq*1+16], 1 - add srcq, r6 + vinserti128 m6, [r7 +ssq*1+ 0], 1 + vinserti128 m1, [r7 +ssq*1+16], 1 + add r7, r6 shufpd m7, m6, m1, 0x05 PUT_8TAP_HV_H 6, 7, 1 ; 1 5 vpermq m4, m4, q1100 @@ -1885,13 +2387,13 @@ cglobal put_8tap_16bpc, 4, 9, 0, dst, ds, src, ss, w, h, mx, my pmaddwd m6, m10 ; b2 paddd m8, m5 paddd m9, m6 - movu xm5, [srcq+ssq*0] - vinserti128 m5, [srcq+ssq*1], 1 + movu xm5, [r7+ssq*0] + vinserti128 m5, [r7+ssq*1], 1 vbroadcasti128 m7, [subpel_h_shufA] vbroadcasti128 m10, [subpel_h_shufB] - movu xm6, [srcq+ssq*0+16] - vinserti128 m6, [srcq+ssq*1+16], 1 - vextracti128 [dstq], m0, 1 + movu xm6, [r7+ssq*0+16] + vinserti128 m6, [r7+ssq*1+16], 1 + vextracti128 [r8], m0, 1 pshufb m0, m5, m7 ; 01 pshufb m5, m10 ; 23 pmaddwd m0, m11 @@ -1902,9 +2404,9 @@ cglobal put_8tap_16bpc, 4, 9, 0, dst, ds, src, ss, w, h, mx, my pmaddwd m5, m13 pmaddwd m6, m14 paddd m6, m5 - movu xm5, [srcq+ssq*0+8] - vinserti128 m5, [srcq+ssq*1+8], 1 - lea srcq, [srcq+ssq*2] + movu xm5, [r7+ssq*0+8] + vinserti128 m5, [r7+ssq*1+8], 1 + lea r7, [r7+ssq*2] pshufb m7, m5, m7 pshufb m5, m10 pmaddwd m10, m13, m7 @@ -1916,7 +2418,7 @@ cglobal put_8tap_16bpc, 4, 9, 0, dst, ds, src, ss, w, h, mx, my pmaddwd m5, m12 paddd m0, m7 paddd m5, m6 - vbroadcasti128 m6, [dstq] + vbroadcasti128 m6, [r8] paddd m8, m10 paddd m9, m10 paddd m0, m10 @@ -1938,16 +2440,14 @@ cglobal put_8tap_16bpc, 4, 9, 0, dst, ds, src, ss, w, h, mx, my packusdw m7, m9 pminsw m7, m15 vpermq m7, m7, q3120 - mova [dstq+dsq*0], xm7 - vextracti128 [dstq+dsq*1], m7, 1 - lea dstq, [dstq+dsq*2] + mova [r8+dsq*0], xm7 + vextracti128 [r8+dsq*1], m7, 1 + lea r8, [r8+dsq*2] sub hd, 2 jg .hv_w8_loop - add r7, 16 - add r8, 16 + add srcq, 16 + add dstq, 16 movzx hd, wb - mov srcq, r7 - mov dstq, r8 sub wd, 1<<8 jg .hv_w8_loop0 RET @@ -1959,28 +2459,24 @@ DECLARE_REG_TMP 6, 7 %endif %define PREP_8TAP_FN FN prep_8tap, -PREP_8TAP_FN sharp, SHARP, SHARP -PREP_8TAP_FN sharp_smooth, SHARP, SMOOTH -PREP_8TAP_FN smooth_sharp, SMOOTH, SHARP -PREP_8TAP_FN smooth, SMOOTH, SMOOTH -PREP_8TAP_FN sharp_regular, SHARP, REGULAR -PREP_8TAP_FN regular_sharp, REGULAR, SHARP -PREP_8TAP_FN smooth_regular, SMOOTH, REGULAR -PREP_8TAP_FN regular_smooth, REGULAR, SMOOTH +PREP_8TAP_FN smooth, SMOOTH, SMOOTH, prep_6tap_16bpc +PREP_8TAP_FN smooth_regular, SMOOTH, REGULAR, prep_6tap_16bpc +PREP_8TAP_FN regular_smooth, REGULAR, SMOOTH, prep_6tap_16bpc PREP_8TAP_FN regular, REGULAR, REGULAR -cglobal prep_8tap_16bpc, 4, 8, 0, tmp, src, stride, w, h, mx, my +cglobal prep_6tap_16bpc, 4, 8, 0, tmp, src, ss, w, h, mx, my %define base r7-prep_avx2 imul mxd, mxm, 0x010101 - add mxd, t0d ; 8tap_h, mx, 4tap_h + add mxd, t0d ; 6tap_h, mx, 4tap_h imul myd, mym, 0x010101 - add myd, t1d ; 8tap_v, my, 4tap_v + add myd, t1d ; 6tap_v, my, 4tap_v lea r7, [prep_avx2] movifnidn hd, hm test mxd, 0xf00 jnz .h test myd, 0xf00 jnz .v +.prep: tzcnt wd, wd mov r6d, r7m ; bitdepth_max movzx wd, word [r7+wq*2+table_offset(prep,)] @@ -1988,7 +2484,7 @@ cglobal prep_8tap_16bpc, 4, 8, 0, tmp, src, stride, w, h, mx, my shr r6d, 11 add wq, r7 vpbroadcastd m4, [base+prep_mul+r6*4] - lea r6, [strideq*3] + lea r6, [ssq*3] %if WIN64 pop r7 %endif @@ -1998,6 +2494,7 @@ cglobal prep_8tap_16bpc, 4, 8, 0, tmp, src, stride, w, h, mx, my sub srcq, 2 pmovsxbw xm0, [base+subpel_filters+mxq*8] vbroadcasti128 m3, [subpel_h_shufA] + lea r6, [ssq*3] vbroadcasti128 m4, [subpel_h_shufB] WIN64_SPILL_XMM 8 pshufd xm0, xm0, q2211 @@ -2008,11 +2505,11 @@ cglobal prep_8tap_16bpc, 4, 8, 0, tmp, src, stride, w, h, mx, my vpbroadcastq m6, xm0 vpermq m7, m0, q1111 .h_w4_loop: - movu xm1, [srcq+strideq*0] - vinserti128 m1, [srcq+strideq*2], 1 - movu xm2, [srcq+strideq*1] - vinserti128 m2, [srcq+r6 ], 1 - lea srcq, [srcq+strideq*4] + movu xm1, [srcq+ssq*0] + vinserti128 m1, [srcq+ssq*2], 1 + movu xm2, [srcq+ssq*1] + vinserti128 m2, [srcq+r6 *1], 1 + lea srcq, [srcq+ssq*4] pshufb m0, m1, m3 ; 0 1 1 2 2 3 3 4 pshufb m1, m4 ; 2 3 3 4 4 5 5 6 pmaddwd m0, m6 @@ -2037,62 +2534,54 @@ cglobal prep_8tap_16bpc, 4, 8, 0, tmp, src, stride, w, h, mx, my test myd, 0xf00 jnz .hv vpbroadcastd m5, [prep_8tap_1d_rnd] ; 8 - (8192 << 4) - lea r6, [strideq*3] cmp wd, 4 je .h_w4 shr mxd, 16 - sub srcq, 6 - vpbroadcastq m0, [base+subpel_filters+mxq*8] - WIN64_SPILL_XMM 12 + sub srcq, 4 + vpbroadcastq m0, [base+subpel_filters+1+mxq*8] + WIN64_SPILL_XMM 10 vbroadcasti128 m6, [subpel_h_shufA] - vbroadcasti128 m7, [subpel_h_shufB] punpcklbw m0, m0 psraw m0, 8 ; sign-extend test dword r7m, 0x800 jnz .h_12bpc psllw m0, 2 .h_12bpc: - pshufd m8, m0, q0000 - pshufd m9, m0, q1111 - pshufd m10, m0, q2222 - pshufd m11, m0, q3333 + pshufd m7, m0, q0000 + pshufd m8, m0, q1111 + pshufd m9, m0, q2222 cmp wd, 8 jg .h_w16 .h_w8: -%macro PREP_8TAP_H 5 ; dst/src+0, src+8, src+16, tmp[1-2] - pshufb m%4, m%1, m7 ; 2 3 3 4 4 5 5 6 - pshufb m%1, m6 ; 0 1 1 2 2 3 3 4 - pmaddwd m%5, m9, m%4 ; abcd1 - pmaddwd m%1, m8 ; abcd0 - pshufb m%2, m7 ; 6 7 7 8 8 9 9 a - shufpd m%4, m%2, 0x05 ; 4 5 5 6 6 7 7 8 - paddd m%5, m5 - paddd m%1, m%5 - pmaddwd m%5, m11, m%2 ; abcd3 - paddd m%1, m%5 - pmaddwd m%5, m10, m%4 ; abcd2 - pshufb m%3, m7 ; a b b c c d d e - pmaddwd m%4, m8 ; efgh0 - paddd m%1, m%5 - pmaddwd m%5, m9, m%2 ; efgh1 - shufpd m%2, m%3, 0x05 ; 8 9 9 a a b b c - pmaddwd m%3, m11 ; efgh3 - pmaddwd m%2, m10 ; efgh2 + movu xm0, [srcq+ssq*0+ 0] + vinserti128 m0, [srcq+ssq*1+ 0], 1 + movu xm2, [srcq+ssq*0+16] + vinserti128 m2, [srcq+ssq*1+16], 1 + lea srcq, [srcq+ssq*2] + shufpd m1, m0, m2, 0x05 +%macro PREP_6TAP_H 5 ; dst/src+0, src+8, src+16, tmp[1-2] + pshufb m%1, m6 ; 01 12 23 34 + pshufb m%2, m6 ; 45 56 67 78 + pmaddwd m%4, m7, m%1 ; a0 + pshufb m%3, m6 ; 89 9a ab bc + pmaddwd m%5, m9, m%2 ; a2 + shufpd m%1, m%2, 0x05 ; 23 34 45 56 + paddd m%4, m%5 ; a0+a2 + pmaddwd m%5, m7, m%2 ; b0 + shufpd m%2, m%3, 0x05 ; 67 78 89 9a + pmaddwd m%3, m9 ; b2 + pmaddwd m%1, m8 ; a1 + pmaddwd m%2, m8 ; b1 + paddd m%3, m%5 ; b0+b2 paddd m%4, m5 - paddd m%4, m%5 - paddd m%3, m%4 + paddd m%3, m5 + paddd m%1, m%4 paddd m%2, m%3 psrad m%1, 4 psrad m%2, 4 packssdw m%1, m%2 %endmacro - movu xm0, [srcq+strideq*0+ 0] - vinserti128 m0, [srcq+strideq*1+ 0], 1 - movu xm2, [srcq+strideq*0+16] - vinserti128 m2, [srcq+strideq*1+16], 1 - lea srcq, [srcq+strideq*2] - shufpd m1, m0, m2, 0x05 - PREP_8TAP_H 0, 1, 2, 3, 4 + PREP_6TAP_H 0, 1, 2, 3, 4 mova [tmpq], m0 add tmpq, 32 sub hd, 2 @@ -2106,11 +2595,11 @@ cglobal prep_8tap_16bpc, 4, 8, 0, tmp, src, stride, w, h, mx, my movu m0, [srcq+r6-32] movu m1, [srcq+r6-24] movu m2, [srcq+r6-16] - PREP_8TAP_H 0, 1, 2, 3, 4 + PREP_6TAP_H 0, 1, 2, 3, 4 mova [tmpq+r6-32], m0 sub r6d, 32 jg .h_w16_loop - add srcq, strideq + add srcq, ssq add tmpq, wq dec hd jg .h_w16_loop0 @@ -2119,13 +2608,368 @@ cglobal prep_8tap_16bpc, 4, 8, 0, tmp, src, stride, w, h, mx, my movzx mxd, myb shr myd, 16 cmp hd, 4 - cmovle myd, mxd + cmove myd, mxd + vpbroadcastq m0, [base+subpel_filters+1+myq*8] + WIN64_SPILL_XMM 9, 12 + vpbroadcastd m5, [prep_8tap_1d_rnd] + mov r6, ssq + punpcklbw m0, m0 + neg r6 + psraw m0, 8 ; sign-extend + test dword r7m, 0x800 + jnz .v_12bpc + psllw m0, 2 +.v_12bpc: + pshufd m6, m0, q0000 + pshufd m7, m0, q1111 + pshufd m8, m0, q2222 + cmp wd, 4 + jg .v_w8 +.v_w4: + movq xm1, [srcq+r6 *2] + vpbroadcastq m3, [srcq+r6 *1] + vpbroadcastq m2, [srcq+ssq*0] + vpbroadcastq m4, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + vpbroadcastq m0, [srcq+ssq*0] + vpblendd m1, m3, 0x30 + vpblendd m3, m2, 0x30 + punpcklwd m1, m3 ; 01 12 + vpblendd m2, m4, 0x30 + vpblendd m4, m0, 0x30 + punpcklwd m2, m4 ; 23 34 +.v_w4_loop: + vpbroadcastq m3, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + pmaddwd m4, m6, m1 ; a0 b0 + mova m1, m2 + pmaddwd m2, m7 ; a1 b1 + paddd m4, m2 + vpblendd m2, m0, m3, 0x30 + vpbroadcastq m0, [srcq+ssq*0] + vpblendd m3, m0, 0x30 + punpcklwd m2, m3 ; 45 56 + pmaddwd m3, m8, m2 ; a2 b2 + paddd m4, m5 + paddd m4, m3 + psrad m4, 4 + vextracti128 xm3, m4, 1 + packssdw xm4, xm3 + mova [tmpq], xm4 + add tmpq, 16 + sub hd, 2 + jg .v_w4_loop + RET +.v_w8: + WIN64_PUSH_XMM 12 +%if WIN64 + push r8 +%endif + mov r8d, wd + shl wd, 5 + lea wd, [hq+wq-256] +.v_w8_loop0: + vbroadcasti128 m3, [srcq+r6 *2] + vbroadcasti128 m4, [srcq+r6 *1] + lea r5, [srcq+ssq*2] + vbroadcasti128 m0, [srcq+ssq*0] + vbroadcasti128 m1, [srcq+ssq*1] + mov r7, tmpq + vbroadcasti128 m2, [r5+ssq*0] + shufpd m3, m0, 0x0c + shufpd m4, m1, 0x0c + punpcklwd m1, m3, m4 ; 01 + punpckhwd m3, m4 ; 23 + shufpd m0, m2, 0x0c + punpcklwd m2, m4, m0 ; 12 + punpckhwd m4, m0 ; 34 +.v_w8_loop: + vbroadcasti128 m9, [r5+ssq*1] + pmaddwd m10, m6, m1 ; a0 + lea r5, [r5+ssq*2] + pmaddwd m11, m6, m2 ; b0 + mova m1, m3 + pmaddwd m3, m7 ; a1 + mova m2, m4 + pmaddwd m4, m7 ; b1 + paddd m10, m5 + paddd m11, m5 + paddd m10, m3 + vbroadcasti128 m3, [r5+ssq*0] + paddd m11, m4 + shufpd m4, m0, m9, 0x0d + shufpd m0, m9, m3, 0x0c + punpcklwd m3, m4, m0 ; 45 + punpckhwd m4, m0 ; 56 + pmaddwd m9, m8, m3 ; a2 + paddd m10, m9 + pmaddwd m9, m8, m4 ; b2 + paddd m11, m9 + psrad m10, 4 + psrad m11, 4 + packssdw m10, m11 + vpermq m10, m10, q3120 + mova [r7+r8*0], xm10 + vextracti128 [r7+r8*2], m10, 1 + lea r7, [r7+r8*4] + sub hd, 2 + jg .v_w8_loop + add srcq, 16 + add tmpq, 16 + movzx hd, wb + sub wd, 1<<8 + jg .v_w8_loop0 +%if WIN64 + pop r8 +%endif + RET +.hv: + WIN64_SPILL_XMM 13, 15 + vpbroadcastd m7, [prep_8tap_2d_rnd] + vbroadcasti128 m8, [subpel_h_shufA] + cmp wd, 4 + jg .hv_w8 + movzx mxd, mxb + vpbroadcastd m0, [base+subpel_filters+mxq*8+2] + movzx mxd, myb + shr myd, 16 + cmp hd, 4 + cmove myd, mxd + vpbroadcastq m1, [base+subpel_filters+1+myq*8] + mov r6, ssq + sub srcq, 2 + pxor m6, m6 + neg r6 + punpcklbw m6, m0 + punpcklbw m1, m1 + psraw m6, 4 + psraw m1, 8 + test dword r7m, 0x800 + jz .hv_w4_10bit + psraw m6, 2 +.hv_w4_10bit: + pshufd m10, m1, q0000 + pshufd m11, m1, q1111 + pshufd m12, m1, q2222 +.hv_w4: + movu xm2, [srcq+r6 *2] + vinserti128 m2, [srcq+r6 *1], 1 ; 0 1 + pshufd m5, m6, q0000 + vbroadcasti128 m9, [base+subpel_h_shufB] + movu xm0, [srcq+ssq*0] + pshufd m6, m6, q1111 + vinserti128 m0, [srcq+ssq*1], 1 ; 2 3 + lea srcq, [srcq+ssq*2] + movu xm3, [srcq+ssq*0] ; 4 + pshufb m1, m2, m8 + pmaddwd m1, m5 + pshufb m2, m9 + pmaddwd m2, m6 + pshufb m4, m0, m8 + pmaddwd m4, m5 + pshufb m0, m9 + pmaddwd m0, m6 + paddd m2, m1 + pshufb xm1, xm3, xm8 + pmaddwd xm1, xm5 + pshufb xm3, xm9 + pmaddwd xm3, xm6 + paddd m0, m4 + paddd m2, m7 + paddd xm1, xm7 + paddd m0, m7 + paddd xm3, xm1 + REPX {psrad x, 6}, m2, m0, xm3 + packssdw m2, m0 ; 0 2 1 3 + packssdw xm0, xm3 ; 2 4 + vperm2i128 m0, m2, 0x03 + punpcklwd m1, m2, m0 ; 01 12 + punpckhwd m2, m0 ; 23 34 +.hv_w4_loop: + movu xm3, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + vinserti128 m3, [srcq+ssq*0], 1 + pmaddwd m4, m10, m1 ; a0 b0 + mova m1, m2 + pmaddwd m2, m11 ; a1 b1 + paddd m4, m2 + pshufb m2, m3, m8 + pmaddwd m2, m5 + pshufb m3, m9 + pmaddwd m3, m6 + paddd m2, m7 + paddd m3, m2 + psrad m3, 6 + packssdw m3, m3 ; 5 5 6 6 + vperm2i128 m2, m0, m3, 0x21 + mova m0, m3 + punpckhwd m2, m3 ; 45 56 + pmaddwd m3, m12, m2 ; a2 b2 + paddd m4, m7 + paddd m4, m3 + psrad m4, 6 + vextracti128 xm3, m4, 1 + packssdw xm4, xm3 + mova [tmpq], xm4 + add tmpq, 16 + sub hd, 2 + jg .hv_w4_loop + RET +.hv_w8: + shr mxd, 16 + vpbroadcastq m2, [base+subpel_filters+1+mxq*8] + movzx mxd, myb + shr myd, 16 + cmp hd, 4 + cmove myd, mxd + pmovsxbw xm1, [base+subpel_filters+1+myq*8] + WIN64_PUSH_XMM 15 +%if WIN64 + PUSH r8 +%endif + mov r8d, wd + shl wd, 5 + mov r6, ssq + sub srcq, 4 + neg r6 + lea wd, [hq+wq-256] + pxor m0, m0 + punpcklbw m0, m2 + psraw m0, 4 + test dword r7m, 0x800 + jz .hv_w8_10bit + psraw m0, 2 +.hv_w8_10bit: + pshufd m10, m0, q0000 + pshufd m11, m0, q1111 + mova [v_mul], xm1 + pshufd m12, m0, q2222 +.hv_w8_loop0: + vbroadcasti128 m0, [srcq+ssq*0+ 0] + vinserti128 m3, m0, [srcq+r6*2+ 0], 0 + lea r5, [srcq+ssq*2] + vbroadcasti128 m2, [srcq+ssq*0+16] + vinserti128 m1, m2, [srcq+r6*2+16], 0 + mov r7, tmpq + vinserti128 m0, [r5 +ssq*0+ 0], 1 + vinserti128 m2, [r5 +ssq*0+16], 1 + shufpd m4, m3, m1, 0x05 +%macro PREP_6TAP_HV_H 5 ; dst/src+0, src+8, src+16, tmp[1-2] + pshufb m%1, m8 ; 01 12 23 34 + pshufb m%2, m8 ; 45 56 67 78 + pmaddwd m%4, m10, m%1 ; a0 + pshufb m%3, m8 ; 89 9a ab bc + pmaddwd m%5, m12, m%2 ; a2 + shufpd m%1, m%2, 0x05 ; 23 34 45 56 + paddd m%4, m%5 ; a0+a2 + pmaddwd m%5, m10, m%2 ; b0 + shufpd m%2, m%3, 0x05 ; 67 78 89 9a + pmaddwd m%3, m12 ; b2 + pmaddwd m%1, m11 ; a1 + pmaddwd m%2, m11 ; b1 + paddd m%3, m%5 ; b0+b2 + paddd m%4, m7 + paddd m%3, m7 + paddd m%1, m%4 + paddd m%2, m%3 + psrad m%1, 6 + psrad m%2, 6 + packssdw m%1, m%2 +%endmacro + PREP_6TAP_HV_H 3, 4, 1, 5, 6 ; 0 2 + movu xm4, [srcq+r6 *1+ 0] + vinserti128 m4, [srcq+ssq*1+ 0], 1 + shufpd m1, m0, m2, 0x05 + PREP_6TAP_HV_H 0, 1, 2, 5, 6 ; 2 4 + movu xm2, [srcq+r6 *1+16] + vinserti128 m2, [srcq+ssq*1+16], 1 + shufpd m1, m4, m2, 0x05 + PREP_6TAP_HV_H 4, 1, 2, 5, 6 ; 1 3 + vpermq m3, m3, q3120 + vpermq m4, m4, q3120 + vpermq m0, m0, q3120 + punpcklwd m1, m3, m4 ; 01 + punpckhwd m3, m4 ; 23 + punpcklwd m2, m4, m0 ; 12 + punpckhwd m4, m0 ; 34 +.hv_w8_loop: + vpbroadcastd m14, [v_mul+4*0] + vpbroadcastd m9, [v_mul+4*1] + movu xm5, [r5+ssq*1+ 0] + movu xm6, [r5+ssq*1+16] + lea r5, [r5+ssq*2] + pmaddwd m13, m14, m1 ; a0 + pmaddwd m14, m2 ; b0 + vinserti128 m5, [r5+ssq*0+ 0], 1 + vinserti128 m6, [r5+ssq*0+16], 1 + mova m1, m3 + pmaddwd m3, m9 ; a1 + mova m2, m4 + pmaddwd m4, m9 ; b1 + paddd m13, m3 + shufpd m3, m5, m6, 0x05 + paddd m14, m4 + PREP_6TAP_HV_H 5, 3, 6, 4, 9 ; 5 6 + vpbroadcastd m6, [v_mul+4*2] + vpermq m5, m5, q3120 + shufpd m4, m0, m5, 0x05 + mova m0, m5 + punpcklwd m3, m4, m5 ; 45 + punpckhwd m4, m5 ; 56 + pmaddwd m5, m6, m3 ; a2 + pmaddwd m6, m4 ; b2 + paddd m13, m7 + paddd m14, m7 + paddd m5, m13 + paddd m6, m14 + psrad m5, 6 + psrad m6, 6 + packssdw m5, m6 + vpermq m5, m5, q3120 + mova [r7+r8*0], xm5 + vextracti128 [r7+r8*2], m5, 1 + lea r7, [r7+r8*4] + sub hd, 2 + jg .hv_w8_loop + add srcq, 16 + add tmpq, 16 + movzx hd, wb + sub wd, 1<<8 + jg .hv_w8_loop0 +%if WIN64 + POP r8 +%endif + RET + +PREP_8TAP_FN smooth_sharp, SMOOTH, SHARP, prep_8tap_16bpc +PREP_8TAP_FN sharp_smooth, SHARP, SMOOTH, prep_8tap_16bpc +PREP_8TAP_FN regular_sharp, REGULAR, SHARP, prep_8tap_16bpc +PREP_8TAP_FN sharp_regular, SHARP, REGULAR, prep_8tap_16bpc +PREP_8TAP_FN sharp, SHARP, SHARP + +cglobal prep_8tap_16bpc, 4, 8, 0, tmp, src, stride, w, h, mx, my +%define base r7-prep_avx2 + imul mxd, mxm, 0x010101 + add mxd, t0d ; 8tap_h, mx, 4tap_h + imul myd, mym, 0x010101 + add myd, t1d ; 8tap_v, my, 4tap_v + lea r7, [prep_avx2] + movifnidn hd, hm + test mxd, 0xf00 + jnz .h + test myd, 0xf00 + jz mangle(private_prefix %+ _prep_6tap_16bpc_avx2).prep +.v: + movzx mxd, myb + shr myd, 16 + cmp hd, 4 + cmove myd, mxd vpbroadcastq m0, [base+subpel_filters+myq*8] - WIN64_SPILL_XMM 15 + WIN64_SPILL_XMM 12, 15 vpbroadcastd m7, [prep_8tap_1d_rnd] lea r6, [strideq*3] - sub srcq, r6 punpcklbw m0, m0 + sub srcq, r6 psraw m0, 8 ; sign-extend test dword r7m, 0x800 jnz .v_12bpc @@ -2183,23 +3027,23 @@ cglobal prep_8tap_16bpc, 4, 8, 0, tmp, src, stride, w, h, mx, my RET .v_w8: %if WIN64 + WIN64_PUSH_XMM 15 push r8 %endif mov r8d, wd shl wd, 5 - mov r5, srcq - mov r7, tmpq lea wd, [hq+wq-256] .v_w8_loop0: vbroadcasti128 m4, [srcq+strideq*0] vbroadcasti128 m5, [srcq+strideq*1] + lea r5, [srcq+strideq*4] vbroadcasti128 m0, [srcq+r6 ] vbroadcasti128 m6, [srcq+strideq*2] - lea srcq, [srcq+strideq*4] - vbroadcasti128 m1, [srcq+strideq*0] - vbroadcasti128 m2, [srcq+strideq*1] - vbroadcasti128 m3, [srcq+strideq*2] - add srcq, r6 + mov r7, tmpq + vbroadcasti128 m1, [r5+strideq*0] + vbroadcasti128 m2, [r5+strideq*1] + vbroadcasti128 m3, [r5+strideq*2] + add r5, r6 shufpd m4, m0, 0x0c shufpd m5, m1, 0x0c punpcklwd m1, m4, m5 ; 01 @@ -2211,7 +3055,7 @@ cglobal prep_8tap_16bpc, 4, 8, 0, tmp, src, stride, w, h, mx, my punpcklwd m3, m6, m0 ; 23 punpckhwd m6, m0 ; 56 .v_w8_loop: - vbroadcasti128 m14, [srcq+strideq*0] + vbroadcasti128 m14, [r5+strideq*0] pmaddwd m12, m8, m1 ; a0 pmaddwd m13, m8, m2 ; b0 mova m1, m3 @@ -2227,8 +3071,8 @@ cglobal prep_8tap_16bpc, 4, 8, 0, tmp, src, stride, w, h, mx, my pmaddwd m5, m10 ; a2 pmaddwd m6, m10 ; b2 paddd m12, m5 - vbroadcasti128 m5, [srcq+strideq*1] - lea srcq, [srcq+strideq*2] + vbroadcasti128 m5, [r5+strideq*1] + lea r5, [r5+strideq*2] paddd m13, m6 shufpd m6, m0, m14, 0x0d shufpd m0, m14, m5, 0x0c @@ -2242,22 +3086,101 @@ cglobal prep_8tap_16bpc, 4, 8, 0, tmp, src, stride, w, h, mx, my psrad m13, 4 packssdw m12, m13 vpermq m12, m12, q3120 - mova [tmpq+r8*0], xm12 - vextracti128 [tmpq+r8*2], m12, 1 - lea tmpq, [tmpq+r8*4] + mova [r7+r8*0], xm12 + vextracti128 [r7+r8*2], m12, 1 + lea r7, [r7+r8*4] sub hd, 2 jg .v_w8_loop - add r5, 16 - add r7, 16 + add srcq, 16 + add tmpq, 16 movzx hd, wb - mov srcq, r5 - mov tmpq, r7 sub wd, 1<<8 jg .v_w8_loop0 %if WIN64 pop r8 %endif RET +.h: + test myd, 0xf00 + jnz .hv + vpbroadcastd m5, [prep_8tap_1d_rnd] ; 8 - (8192 << 4) + cmp wd, 4 + je mangle(private_prefix %+ _prep_6tap_16bpc_avx2).h_w4 + shr mxd, 16 + sub srcq, 6 + vpbroadcastq m0, [base+subpel_filters+mxq*8] + WIN64_SPILL_XMM 12 + vbroadcasti128 m6, [subpel_h_shufA] + vbroadcasti128 m7, [subpel_h_shufB] + punpcklbw m0, m0 + psraw m0, 8 ; sign-extend + test dword r7m, 0x800 + jnz .h_12bpc + psllw m0, 2 +.h_12bpc: + pshufd m8, m0, q0000 + pshufd m9, m0, q1111 + pshufd m10, m0, q2222 + pshufd m11, m0, q3333 + cmp wd, 8 + jg .h_w16 +.h_w8: +%macro PREP_8TAP_H 5 ; dst/src+0, src+8, src+16, tmp[1-2] + pshufb m%4, m%1, m7 ; 2 3 3 4 4 5 5 6 + pshufb m%1, m6 ; 0 1 1 2 2 3 3 4 + pmaddwd m%5, m9, m%4 ; abcd1 + pmaddwd m%1, m8 ; abcd0 + pshufb m%2, m7 ; 6 7 7 8 8 9 9 a + shufpd m%4, m%2, 0x05 ; 4 5 5 6 6 7 7 8 + paddd m%5, m5 + paddd m%1, m%5 + pmaddwd m%5, m11, m%2 ; abcd3 + paddd m%1, m%5 + pmaddwd m%5, m10, m%4 ; abcd2 + pshufb m%3, m7 ; a b b c c d d e + pmaddwd m%4, m8 ; efgh0 + paddd m%1, m%5 + pmaddwd m%5, m9, m%2 ; efgh1 + shufpd m%2, m%3, 0x05 ; 8 9 9 a a b b c + pmaddwd m%3, m11 ; efgh3 + pmaddwd m%2, m10 ; efgh2 + paddd m%4, m5 + paddd m%4, m%5 + paddd m%3, m%4 + paddd m%2, m%3 + psrad m%1, 4 + psrad m%2, 4 + packssdw m%1, m%2 +%endmacro + movu xm0, [srcq+strideq*0+ 0] + vinserti128 m0, [srcq+strideq*1+ 0], 1 + movu xm2, [srcq+strideq*0+16] + vinserti128 m2, [srcq+strideq*1+16], 1 + lea srcq, [srcq+strideq*2] + shufpd m1, m0, m2, 0x05 + PREP_8TAP_H 0, 1, 2, 3, 4 + mova [tmpq], m0 + add tmpq, 32 + sub hd, 2 + jg .h_w8 + RET +.h_w16: + add wd, wd +.h_w16_loop0: + mov r6d, wd +.h_w16_loop: + movu m0, [srcq+r6-32] + movu m1, [srcq+r6-24] + movu m2, [srcq+r6-16] + PREP_8TAP_H 0, 1, 2, 3, 4 + mova [tmpq+r6-32], m0 + sub r6d, 32 + jg .h_w16_loop + add srcq, strideq + add tmpq, wq + dec hd + jg .h_w16_loop0 + RET .hv: WIN64_SPILL_XMM 16 vpbroadcastd m15, [prep_8tap_2d_rnd] @@ -2268,12 +3191,12 @@ cglobal prep_8tap_16bpc, 4, 8, 0, tmp, src, stride, w, h, mx, my movzx mxd, myb shr myd, 16 cmp hd, 4 - cmovle myd, mxd + cmove myd, mxd vpbroadcastq m1, [base+subpel_filters+myq*8] lea r6, [strideq*3] sub srcq, 2 - sub srcq, r6 pxor m7, m7 + sub srcq, r6 punpcklbw m7, m0 punpcklbw m1, m1 psraw m7, 4 @@ -2375,7 +3298,7 @@ cglobal prep_8tap_16bpc, 4, 8, 0, tmp, src, stride, w, h, mx, my movzx mxd, myb shr myd, 16 cmp hd, 4 - cmovle myd, mxd + cmove myd, mxd pmovsxbw xm1, [base+subpel_filters+myq*8] %if WIN64 PUSH r8 @@ -2385,12 +3308,9 @@ cglobal prep_8tap_16bpc, 4, 8, 0, tmp, src, stride, w, h, mx, my lea r6, [strideq*3] sub srcq, 6 sub srcq, r6 - mov r5, srcq - mov r7, tmpq lea wd, [hq+wq-256] pxor m0, m0 punpcklbw m0, m2 - mova [v_mul], xm1 psraw m0, 4 test dword r7m, 0x800 jz .hv_w8_10bit @@ -2398,6 +3318,7 @@ cglobal prep_8tap_16bpc, 4, 8, 0, tmp, src, stride, w, h, mx, my .hv_w8_10bit: pshufd m11, m0, q0000 pshufd m12, m0, q1111 + mova [v_mul], xm1 pshufd m13, m0, q2222 pshufd m14, m0, q3333 .hv_w8_loop0: @@ -2430,13 +3351,15 @@ cglobal prep_8tap_16bpc, 4, 8, 0, tmp, src, stride, w, h, mx, my %endmacro movu xm4, [srcq+r6 + 0] vbroadcasti128 m8, [subpel_h_shufA] + lea r5, [srcq+strideq*4] movu xm6, [srcq+r6 + 8] vbroadcasti128 m9, [subpel_h_shufB] + mov r7, tmpq movu xm0, [srcq+r6 +16] movu xm5, [srcq+strideq*0+ 0] - vinserti128 m5, [srcq+strideq*4+ 0], 1 + vinserti128 m5, [r5 +strideq*0+ 0], 1 movu xm1, [srcq+strideq*0+16] - vinserti128 m1, [srcq+strideq*4+16], 1 + vinserti128 m1, [r5 +strideq*0+16], 1 shufpd m7, m5, m1, 0x05 INIT_XMM avx2 PREP_8TAP_HV_H 4, 6, 0 ; 3 @@ -2450,10 +3373,9 @@ cglobal prep_8tap_16bpc, 4, 8, 0, tmp, src, stride, w, h, mx, my PREP_8TAP_HV_H 0, 7, 1 ; 2 6 movu xm6, [srcq+strideq*1+ 0] movu xm1, [srcq+strideq*1+16] - lea srcq, [srcq+strideq*4] - vinserti128 m6, [srcq+strideq*1+ 0], 1 - vinserti128 m1, [srcq+strideq*1+16], 1 - add srcq, r6 + vinserti128 m6, [r5 +strideq*1+ 0], 1 + vinserti128 m1, [r5 +strideq*1+16], 1 + add r5, r6 shufpd m7, m6, m1, 0x05 PREP_8TAP_HV_H 6, 7, 1 ; 1 5 vpermq m4, m4, q1100 @@ -2486,13 +3408,13 @@ cglobal prep_8tap_16bpc, 4, 8, 0, tmp, src, stride, w, h, mx, my pmaddwd m6, m10 ; b2 paddd m8, m5 paddd m9, m6 - movu xm5, [srcq+strideq*0] - vinserti128 m5, [srcq+strideq*1], 1 + movu xm5, [r5+strideq*0] + vinserti128 m5, [r5+strideq*1], 1 vbroadcasti128 m7, [subpel_h_shufA] vbroadcasti128 m10, [subpel_h_shufB] - movu xm6, [srcq+strideq*0+16] - vinserti128 m6, [srcq+strideq*1+16], 1 - vextracti128 [tmpq], m0, 1 + movu xm6, [r5+strideq*0+16] + vinserti128 m6, [r5+strideq*1+16], 1 + vextracti128 [r7], m0, 1 pshufb m0, m5, m7 ; 01 pshufb m5, m10 ; 23 pmaddwd m0, m11 @@ -2505,9 +3427,9 @@ cglobal prep_8tap_16bpc, 4, 8, 0, tmp, src, stride, w, h, mx, my pmaddwd m6, m14 paddd m5, m15 paddd m6, m5 - movu xm5, [srcq+strideq*0+8] - vinserti128 m5, [srcq+strideq*1+8], 1 - lea srcq, [srcq+strideq*2] + movu xm5, [r5+strideq*0+8] + vinserti128 m5, [r5+strideq*1+8], 1 + lea r5, [r5+strideq*2] pshufb m7, m5, m7 pshufb m5, m10 pmaddwd m10, m13, m7 @@ -2518,7 +3440,7 @@ cglobal prep_8tap_16bpc, 4, 8, 0, tmp, src, stride, w, h, mx, my pmaddwd m5, m12 paddd m0, m7 paddd m5, m6 - vbroadcasti128 m6, [tmpq] + vbroadcasti128 m6, [r7] vpbroadcastd m10, [v_mul+4*3] psrad m0, 6 psrad m5, 6 @@ -2535,16 +3457,14 @@ cglobal prep_8tap_16bpc, 4, 8, 0, tmp, src, stride, w, h, mx, my psrad m9, 6 packssdw m7, m9 vpermq m7, m7, q3120 - mova [tmpq+r8*0], xm7 - vextracti128 [tmpq+r8*2], m7, 1 - lea tmpq, [tmpq+r8*4] + mova [r7+r8*0], xm7 + vextracti128 [r7+r8*2], m7, 1 + lea r7, [r7+r8*4] sub hd, 2 jg .hv_w8_loop - add r5, 16 - add r7, 16 + add srcq, 16 + add tmpq, 16 movzx hd, wb - mov srcq, r5 - mov tmpq, r7 sub wd, 1<<8 jg .hv_w8_loop0 %if WIN64 @@ -4223,14 +5143,14 @@ DECLARE_REG_TMP 6, 8 %define PUT_8TAP_SCALED_FN FN put_8tap_scaled, BILIN_SCALED_FN put -PUT_8TAP_SCALED_FN sharp, SHARP, SHARP -PUT_8TAP_SCALED_FN sharp_smooth, SHARP, SMOOTH -PUT_8TAP_SCALED_FN smooth_sharp, SMOOTH, SHARP -PUT_8TAP_SCALED_FN smooth, SMOOTH, SMOOTH -PUT_8TAP_SCALED_FN sharp_regular, SHARP, REGULAR -PUT_8TAP_SCALED_FN regular_sharp, REGULAR, SHARP -PUT_8TAP_SCALED_FN smooth_regular, SMOOTH, REGULAR -PUT_8TAP_SCALED_FN regular_smooth, REGULAR, SMOOTH +PUT_8TAP_SCALED_FN sharp, SHARP, SHARP, put_8tap_scaled_16bpc +PUT_8TAP_SCALED_FN sharp_smooth, SHARP, SMOOTH, put_8tap_scaled_16bpc +PUT_8TAP_SCALED_FN smooth_sharp, SMOOTH, SHARP, put_8tap_scaled_16bpc +PUT_8TAP_SCALED_FN smooth, SMOOTH, SMOOTH, put_8tap_scaled_16bpc +PUT_8TAP_SCALED_FN sharp_regular, SHARP, REGULAR, put_8tap_scaled_16bpc +PUT_8TAP_SCALED_FN regular_sharp, REGULAR, SHARP, put_8tap_scaled_16bpc +PUT_8TAP_SCALED_FN smooth_regular, SMOOTH, REGULAR, put_8tap_scaled_16bpc +PUT_8TAP_SCALED_FN regular_smooth, REGULAR, SMOOTH, put_8tap_scaled_16bpc PUT_8TAP_SCALED_FN regular, REGULAR, REGULAR MC_8TAP_SCALED put @@ -4242,14 +5162,14 @@ DECLARE_REG_TMP 6, 7 %define PREP_8TAP_SCALED_FN FN prep_8tap_scaled, BILIN_SCALED_FN prep -PREP_8TAP_SCALED_FN sharp, SHARP, SHARP -PREP_8TAP_SCALED_FN sharp_smooth, SHARP, SMOOTH -PREP_8TAP_SCALED_FN smooth_sharp, SMOOTH, SHARP -PREP_8TAP_SCALED_FN smooth, SMOOTH, SMOOTH -PREP_8TAP_SCALED_FN sharp_regular, SHARP, REGULAR -PREP_8TAP_SCALED_FN regular_sharp, REGULAR, SHARP -PREP_8TAP_SCALED_FN smooth_regular, SMOOTH, REGULAR -PREP_8TAP_SCALED_FN regular_smooth, REGULAR, SMOOTH +PREP_8TAP_SCALED_FN sharp, SHARP, SHARP, prep_8tap_scaled_16bpc +PREP_8TAP_SCALED_FN sharp_smooth, SHARP, SMOOTH, prep_8tap_scaled_16bpc +PREP_8TAP_SCALED_FN smooth_sharp, SMOOTH, SHARP, prep_8tap_scaled_16bpc +PREP_8TAP_SCALED_FN smooth, SMOOTH, SMOOTH, prep_8tap_scaled_16bpc +PREP_8TAP_SCALED_FN sharp_regular, SHARP, REGULAR, prep_8tap_scaled_16bpc +PREP_8TAP_SCALED_FN regular_sharp, REGULAR, SHARP, prep_8tap_scaled_16bpc +PREP_8TAP_SCALED_FN smooth_regular, SMOOTH, REGULAR, prep_8tap_scaled_16bpc +PREP_8TAP_SCALED_FN regular_smooth, REGULAR, SMOOTH, prep_8tap_scaled_16bpc PREP_8TAP_SCALED_FN regular, REGULAR, REGULAR MC_8TAP_SCALED prep diff --git a/third_party/dav1d/src/x86/mc_avx2.asm b/third_party/dav1d/src/x86/mc_avx2.asm index 58e3cb5af1..df8bebb1cb 100644 --- a/third_party/dav1d/src/x86/mc_avx2.asm +++ b/third_party/dav1d/src/x86/mc_avx2.asm @@ -60,15 +60,14 @@ subpel_v_shuf4: db 0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 1 subpel_s_shuf2: db 0, 1, 2, 3, 0, 1, 2, 3, 8, 9, 10, 11, 8, 9, 10, 11 subpel_s_shuf8: db 0, 1, 8, 9, 2, 3, 10, 11, 4, 5, 12, 13, 6, 7, 14, 15 bilin_h_shuf4: db 0, 1, 1, 2, 2, 3, 3, 4, 8, 9, 9, 10, 10, 11, 11, 12 -bilin_h_shuf8: db 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8 bilin_v_shuf4: db 4, 0, 5, 1, 6, 2, 7, 3, 8, 4, 9, 5, 10, 6, 11, 7 deint_shuf4: db 0, 4, 1, 5, 2, 6, 3, 7, 4, 8, 5, 9, 6, 10, 7, 11 blend_shuf: db 0, 1, 0, 1, 0, 1, 0, 1, 2, 3, 2, 3, 2, 3, 2, 3 pb_8x0_8x8: db 0, 0, 0, 0, 0, 0, 0, 0, 8, 8, 8, 8, 8, 8, 8, 8 bdct_lb_dw: db 0, 0, 0, 0, 4, 4, 4, 4, 8, 8, 8, 8, 12, 12, 12, 12 wswap: db 2, 3, 0, 1, 6, 7, 4, 5, 10, 11, 8, 9, 14, 15, 12, 13 -rescale_mul: dd 0, 1, 2, 3, 4, 5, 6, 7 resize_shuf: db 0, 0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 7, 7, 7, 7, 7 +rescale_mul: dd 0, 1, 2, 3, 4, 5, 6, 7 wm_420_sign: dd 0x01020102, 0x01010101 wm_422_sign: dd 0x80808080, 0x7f7f7f7f @@ -95,6 +94,7 @@ pq_0x40000000: dq 0x40000000 cextern mc_subpel_filters cextern mc_warp_filter2 cextern resize_filter +cextern z_filter_s %define subpel_filters (mangle(private_prefix %+ _mc_subpel_filters)-8) @@ -184,7 +184,9 @@ BASE_JMP_TABLE put, avx2, 2, 4, 8, 16, 32, 64, 128 BASE_JMP_TABLE prep, avx2, 4, 8, 16, 32, 64, 128 HV_JMP_TABLE put, bilin, avx2, 7, 2, 4, 8, 16, 32, 64, 128 HV_JMP_TABLE prep, bilin, avx2, 7, 4, 8, 16, 32, 64, 128 +HV_JMP_TABLE put, 6tap, avx2, 3, 2, 4, 8, 16, 32, 64, 128 HV_JMP_TABLE put, 8tap, avx2, 3, 2, 4, 8, 16, 32, 64, 128 +HV_JMP_TABLE prep, 6tap, avx2, 1, 4, 8, 16, 32, 64, 128 HV_JMP_TABLE prep, 8tap, avx2, 1, 4, 8, 16, 32, 64, 128 SCALED_JMP_TABLE put_8tap_scaled, avx2, 2, 4, 8, 16, 32, 64, 128 SCALED_JMP_TABLE prep_8tap_scaled, avx2, 4, 8, 16, 32, 64, 128 @@ -298,7 +300,7 @@ INIT_YMM avx2 ; (16 * src[x] + (mx * (src[x + 1] - src[x])) + 8) >> 4 ; = ((16 - mx) * src[x] + mx * src[x + 1] + 8) >> 4 imul mxyd, 255 - vbroadcasti128 m4, [bilin_h_shuf8] + vbroadcasti128 m4, [z_filter_s+2] add mxyd, 16 movd xm5, mxyd mov mxyd, r7m ; my @@ -900,7 +902,7 @@ cglobal prep_bilin_8bpc, 3, 7, 0, tmp, src, stride, w, h, mxy, stride3 ; 16 * src[x] + (mx * (src[x + 1] - src[x])) ; = (16 - mx) * src[x] + mx * src[x + 1] imul mxyd, 255 - vbroadcasti128 m4, [bilin_h_shuf8] + vbroadcasti128 m4, [z_filter_s+2] add mxyd, 16 movd xm5, mxyd mov mxyd, r6m ; my @@ -1436,7 +1438,7 @@ cglobal prep_bilin_8bpc, 3, 7, 0, tmp, src, stride, w, h, mxy, stride3 %assign FILTER_SMOOTH (1*15 << 16) | 4*15 %assign FILTER_SHARP (2*15 << 16) | 3*15 -%macro FN 4 ; fn, type, type_h, type_v +%macro FN 4-5 ; fn, type, type_h, type_v, jmp_to cglobal %1_%2_8bpc mov t0d, FILTER_%3 %ifidn %3, %4 @@ -1444,8 +1446,8 @@ cglobal %1_%2_8bpc %else mov t1d, FILTER_%4 %endif -%ifnidn %2, regular ; skip the jump in the last filter - jmp mangle(private_prefix %+ _%1_8bpc %+ SUFFIX) +%if %0 == 5 ; skip the jump in the last filter + jmp mangle(private_prefix %+ _%5 %+ SUFFIX) %endif %endmacro @@ -1456,28 +1458,24 @@ DECLARE_REG_TMP 7, 8 %endif %define PUT_8TAP_FN FN put_8tap, -PUT_8TAP_FN sharp, SHARP, SHARP -PUT_8TAP_FN sharp_smooth, SHARP, SMOOTH -PUT_8TAP_FN smooth_sharp, SMOOTH, SHARP -PUT_8TAP_FN smooth, SMOOTH, SMOOTH -PUT_8TAP_FN sharp_regular, SHARP, REGULAR -PUT_8TAP_FN regular_sharp, REGULAR, SHARP -PUT_8TAP_FN smooth_regular, SMOOTH, REGULAR -PUT_8TAP_FN regular_smooth, REGULAR, SMOOTH +PUT_8TAP_FN smooth, SMOOTH, SMOOTH, put_6tap_8bpc +PUT_8TAP_FN smooth_regular, SMOOTH, REGULAR, put_6tap_8bpc +PUT_8TAP_FN regular_smooth, REGULAR, SMOOTH, put_6tap_8bpc PUT_8TAP_FN regular, REGULAR, REGULAR -cglobal put_8tap_8bpc, 4, 9, 0, dst, ds, src, ss, w, h, mx, my, ss3 +cglobal put_6tap_8bpc, 4, 9, 0, dst, ds, src, ss, w, h, mx, my, ns imul mxd, mxm, 0x010101 - add mxd, t0d ; 8tap_h, mx, 4tap_h + add mxd, t0d ; 6tap_h, mx, 4tap_h imul myd, mym, 0x010101 - add myd, t1d ; 8tap_v, my, 4tap_v + add myd, t1d ; 6tap_v, my, 4tap_v lea r8, [put_avx2] - movsxd wq, wm + mov wd, wm movifnidn hd, hm test mxd, 0xf00 jnz .h test myd, 0xf00 jnz .v +.put: tzcnt wd, wd movzx wd, word [r8+wq*2+table_offset(put,)] add wq, r8 @@ -1487,36 +1485,18 @@ cglobal put_8tap_8bpc, 4, 9, 0, dst, ds, src, ss, w, h, mx, my, ss3 pop r8 %endif jmp wq -.h: - test myd, 0xf00 - jnz .hv - vpbroadcastd m5, [pw_34] ; 2 + (8 << 2) - WIN64_SPILL_XMM 11 - cmp wd, 4 - jl .h_w2 - vbroadcasti128 m6, [subpel_h_shufA] - je .h_w4 - tzcnt wd, wd - vbroadcasti128 m7, [subpel_h_shufB] - vbroadcasti128 m8, [subpel_h_shufC] - shr mxd, 16 - sub srcq, 3 - movzx wd, word [r8+wq*2+table_offset(put, _8tap_h)] - vpbroadcastd m9, [r8+mxq*8+subpel_filters-put_avx2+0] - vpbroadcastd m10, [r8+mxq*8+subpel_filters-put_avx2+4] - add wq, r8 - jmp wq .h_w2: movzx mxd, mxb - dec srcq - mova xm4, [subpel_h_shuf4] - vpbroadcastd xm3, [r8+mxq*8+subpel_filters-put_avx2+2] + lea srcq, [srcq-1] + vpbroadcastd xm4, [r8+mxq*8+subpel_filters-put_avx2+2] + je .h_w4 + mova xm3, [subpel_h_shuf4] .h_w2_loop: movq xm0, [srcq+ssq*0] movhps xm0, [srcq+ssq*1] lea srcq, [srcq+ssq*2] - pshufb xm0, xm4 - pmaddubsw xm0, xm3 + pshufb xm0, xm3 + pmaddubsw xm0, xm4 phaddw xm0, xm0 paddw xm0, xm5 psraw xm0, 6 @@ -1528,17 +1508,15 @@ cglobal put_8tap_8bpc, 4, 9, 0, dst, ds, src, ss, w, h, mx, my, ss3 jg .h_w2_loop RET .h_w4: - movzx mxd, mxb - dec srcq - vpbroadcastd xm3, [r8+mxq*8+subpel_filters-put_avx2+2] + mova xm3, [subpel_h_shufA] .h_w4_loop: movq xm0, [srcq+ssq*0] movq xm1, [srcq+ssq*1] lea srcq, [srcq+ssq*2] - pshufb xm0, xm6 - pshufb xm1, xm6 - pmaddubsw xm0, xm3 - pmaddubsw xm1, xm3 + pshufb xm0, xm3 + pshufb xm1, xm3 + pmaddubsw xm0, xm4 + pmaddubsw xm1, xm4 phaddw xm0, xm1 paddw xm0, xm5 psraw xm0, 6 @@ -1549,25 +1527,43 @@ cglobal put_8tap_8bpc, 4, 9, 0, dst, ds, src, ss, w, h, mx, my, ss3 sub hd, 2 jg .h_w4_loop RET +.h: + test myd, 0xf00 + jnz .hv + vpbroadcastd m5, [pw_34] ; 2 + (8 << 2) + cmp wd, 4 + jle .h_w2 + WIN64_SPILL_XMM 11 + tzcnt wd, wd + vbroadcasti128 m4, [z_filter_s+ 2] ; 01 + shr mxd, 16 + vbroadcasti128 m6, [z_filter_s+ 6] ; 23 + sub srcq, 2 + vbroadcasti128 m7, [z_filter_s+10] ; 45 + lea mxq, [r8+mxq*8+subpel_filters+1-put_avx2] + movzx wd, word [r8+wq*2+table_offset(put, _6tap_h)] + vpbroadcastw m8, [mxq+0] + vpbroadcastw m9, [mxq+2] + add wq, r8 + vpbroadcastw m10, [mxq+4] + jmp wq .h_w8: -%macro PUT_8TAP_H 4 ; dst/src, tmp[1-3] - pshufb m%2, m%1, m7 - pshufb m%3, m%1, m8 - pshufb m%1, m6 - pmaddubsw m%4, m%2, m9 - pmaddubsw m%2, m10 - pmaddubsw m%3, m10 - pmaddubsw m%1, m9 - paddw m%3, m%4 +%macro PUT_6TAP_H 3 ; dst/src, tmp[1-2] + pshufb m%2, m%1, m4 + pmaddubsw m%2, m8 + pshufb m%3, m%1, m6 + pmaddubsw m%3, m9 + pshufb m%1, m7 + pmaddubsw m%1, m10 + paddw m%2, m5 + paddw m%1, m%3 paddw m%1, m%2 - phaddw m%1, m%3 - paddw m%1, m5 psraw m%1, 6 %endmacro movu xm0, [srcq+ssq*0] vinserti128 m0, [srcq+ssq*1], 1 lea srcq, [srcq+ssq*2] - PUT_8TAP_H 0, 1, 2, 3 + PUT_6TAP_H 0, 1, 2 vextracti128 xm1, m0, 1 packuswb xm0, xm1 movq [dstq+dsq*0], xm0 @@ -1581,9 +1577,9 @@ cglobal put_8tap_8bpc, 4, 9, 0, dst, ds, src, ss, w, h, mx, my, ss3 vinserti128 m0, [srcq+ssq*1+8*0], 1 movu xm1, [srcq+ssq*0+8*1] vinserti128 m1, [srcq+ssq*1+8*1], 1 - PUT_8TAP_H 0, 2, 3, 4 + PUT_6TAP_H 0, 2, 3 lea srcq, [srcq+ssq*2] - PUT_8TAP_H 1, 2, 3, 4 + PUT_6TAP_H 1, 2, 3 packuswb m0, m1 mova [dstq+dsq*0], xm0 vextracti128 [dstq+dsq*1], m0, 1 @@ -1606,8 +1602,8 @@ cglobal put_8tap_8bpc, 4, 9, 0, dst, ds, src, ss, w, h, mx, my, ss3 .h_loop: movu m0, [srcq+r6+8*0] movu m1, [srcq+r6+8*1] - PUT_8TAP_H 0, 2, 3, 4 - PUT_8TAP_H 1, 2, 3, 4 + PUT_6TAP_H 0, 2, 3 + PUT_6TAP_H 1, 2, 3 packuswb m0, m1 mova [dstq+r6], m0 add r6, 32 @@ -1619,7 +1615,421 @@ cglobal put_8tap_8bpc, 4, 9, 0, dst, ds, src, ss, w, h, mx, my, ss3 jg .h_loop RET .v: - WIN64_SPILL_XMM 16 + WIN64_SPILL_XMM 9, 12 + movzx mxd, myb + shr myd, 16 + cmp hd, 6 + cmovs myd, mxd + tzcnt r6d, wd + movzx r6d, word [r8+r6*2+table_offset(put, _6tap_v)] + vpbroadcastd m8, [pw_512] + lea myq, [r8+myq*8+subpel_filters+1-put_avx2] + vpbroadcastw m5, [myq+0] + vpbroadcastw m6, [myq+2] + vpbroadcastw m7, [myq+4] + add r6, r8 + mov nsq, ssq + neg nsq + jmp r6 +.v_w2: + movd xm2, [srcq+nsq*2] + pinsrw xm2, [srcq+nsq*1], 2 + pinsrw xm2, [srcq+ssq*0], 4 + pinsrw xm2, [srcq+ssq*1], 6 ; 0 1 2 3 + lea srcq, [srcq+ssq*2] + vpbroadcastd xm0, [srcq+ssq*0] + palignr xm3, xm0, xm2, 4 ; 1 2 3 4 + punpcklbw xm1, xm2, xm3 ; 01 12 + punpckhbw xm2, xm3 ; 23 34 +.v_w2_loop: + vpbroadcastd xm4, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + pmaddubsw xm3, xm1, xm5 ; a0 b0 + mova xm1, xm2 + pmaddubsw xm2, xm6 ; a1 b1 + paddw xm3, xm2 + vpblendd xm2, xm0, xm4, 0x02 ; 4 5 + vpbroadcastd xm0, [srcq+ssq*0] + vpblendd xm4, xm0, 0x02 ; 5 6 + punpcklbw xm2, xm4 ; 67 78 + pmaddubsw xm4, xm2, xm7 ; a3 b3 + paddw xm3, xm4 + pmulhrsw xm3, xm8 + packuswb xm3, xm3 + pextrw [dstq+dsq*0], xm3, 0 + pextrw [dstq+dsq*1], xm3, 2 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .v_w2_loop + RET +.v_w4: + movd xm2, [srcq+nsq*2] + pinsrd xm2, [srcq+nsq*1], 1 + pinsrd xm2, [srcq+ssq*0], 2 + pinsrd xm2, [srcq+ssq*1], 3 ; 0 1 2 3 + lea srcq, [srcq+ssq*2] + vpbroadcastd xm0, [srcq+ssq*0] + palignr xm3, xm0, xm2, 4 ; 1 2 3 4 + punpcklbw xm1, xm2, xm3 ; 01 12 + punpckhbw xm2, xm3 ; 23 34 +.v_w4_loop: + vpbroadcastd xm4, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + pmaddubsw xm3, xm1, xm5 ; a0 b0 + mova xm1, xm2 + pmaddubsw xm2, xm6 ; a1 b1 + paddw xm3, xm2 + vpblendd xm2, xm0, xm4, 0x02 ; 4 5 + vpbroadcastd xm0, [srcq+ssq*0] + vpblendd xm4, xm0, 0x02 ; 5 6 + punpcklbw xm2, xm4 ; 45 56 + pmaddubsw xm4, xm2, xm7 ; a2 b2 + paddw xm3, xm4 + pmulhrsw xm3, xm8 + packuswb xm3, xm3 + movd [dstq+dsq*0], xm3 + pextrd [dstq+dsq*1], xm3, 1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .v_w4_loop + RET +.v_w8: + movq xm1, [srcq+nsq*2] + vpbroadcastq m3, [srcq+nsq*1] + vpbroadcastq m2, [srcq+ssq*0] + vpbroadcastq m4, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + vpbroadcastq m0, [srcq+ssq*0] + vpblendd m1, m3, 0x30 + vpblendd m3, m2, 0x30 + punpcklbw m1, m3 ; 01 12 + vpblendd m2, m4, 0x30 + vpblendd m4, m0, 0x30 + punpcklbw m2, m4 ; 23 34 +.v_w8_loop: + vpbroadcastq m4, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + pmaddubsw m3, m1, m5 ; a0 b0 + mova m1, m2 + pmaddubsw m2, m6 ; a1 b1 + paddw m3, m2 + vpblendd m2, m0, m4, 0x30 + vpbroadcastq m0, [srcq+ssq*0] + vpblendd m4, m0, 0x30 + punpcklbw m2, m4 ; 45 56 + pmaddubsw m4, m2, m7 ; a2 b2 + paddw m3, m4 + pmulhrsw m3, m8 + vextracti128 xm4, m3, 1 + packuswb xm3, xm4 + movq [dstq+dsq*0], xm3 + movhps [dstq+dsq*1], xm3 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .v_w8_loop + RET +.v_w16: +.v_w32: +.v_w64: +.v_w128: + lea r6d, [wq*8-128] + WIN64_PUSH_XMM 12 + lea r6d, [hq+r6*2] +.v_w16_loop0: + vbroadcasti128 m3, [srcq+nsq*2] + vbroadcasti128 m4, [srcq+nsq*1] + lea r4, [srcq+ssq*2] + vbroadcasti128 m0, [srcq+ssq*0] + vbroadcasti128 m1, [srcq+ssq*1] + mov r7, dstq + vbroadcasti128 m2, [r4+ssq*0] + shufpd m3, m0, 0x0c + shufpd m4, m1, 0x0c + punpcklbw m1, m3, m4 ; 01 + punpckhbw m3, m4 ; 23 + shufpd m0, m2, 0x0c + punpcklbw m2, m4, m0 ; 12 + punpckhbw m4, m0 ; 34 +.v_w16_loop: + vbroadcasti128 m9, [r4+ssq*1] + pmaddubsw m10, m1, m5 ; a0 + lea r4, [r4+ssq*2] + pmaddubsw m11, m2, m5 ; b0 + mova m1, m3 + pmaddubsw m3, m6 ; a1 + mova m2, m4 + pmaddubsw m4, m6 ; b1 + paddw m10, m3 + vbroadcasti128 m3, [r4+ssq*0] + paddw m11, m4 + shufpd m4, m0, m9, 0x0d + shufpd m0, m9, m3, 0x0c + punpcklbw m3, m4, m0 ; 45 + punpckhbw m4, m0 ; 56 + pmaddubsw m9, m3, m7 ; a2 + paddw m10, m9 + pmaddubsw m9, m4, m7 ; b2 + paddw m11, m9 + pmulhrsw m10, m8 + pmulhrsw m11, m8 + packuswb m10, m11 + vpermq m10, m10, q3120 + mova [r7+dsq*0], xm10 + vextracti128 [r7+dsq*1], m10, 1 + lea r7, [r7+dsq*2] + sub hd, 2 + jg .v_w16_loop + add srcq, 16 + add dstq, 16 + movzx hd, r6b + sub r6d, 1<<8 + jg .v_w16_loop0 + RET +.hv: + WIN64_SPILL_XMM 12, 16 + cmp wd, 4 + jg .hv_w8 + movzx mxd, mxb + dec srcq + vpbroadcastd m6, [r8+mxq*8+subpel_filters-put_avx2+2] + movzx mxd, myb + shr myd, 16 + cmp hd, 6 + cmovs myd, mxd + vpbroadcastq m0, [r8+myq*8+subpel_filters+1-put_avx2] + vpbroadcastd m7, [pw_8192] + punpcklbw m0, m0 + vpbroadcastd m8, [pd_512] + psraw m0, 8 ; sign-extend + mov nsq, ssq + pshufd m9, m0, q0000 + neg nsq + pshufd m10, m0, q1111 + pshufd m11, m0, q2222 + cmp wd, 4 + je .hv_w4 + vbroadcasti128 m5, [subpel_h_shuf4] + movq xm2, [srcq+nsq*2] + movhps xm2, [srcq+nsq*1] + movq xm0, [srcq+ssq*0] + movhps xm0, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + vpbroadcastq m1, [srcq+ssq*0] + vpblendd m2, m1, 0x30 + pshufb m2, m5 + pshufb xm0, xm5 + pmaddubsw m2, m6 + pmaddubsw xm0, xm6 + phaddw m2, m0 + pmulhrsw m2, m7 + vextracti128 xm0, m2, 1 + palignr xm0, xm2, 4 + punpcklwd xm1, xm2, xm0 ; 01 12 + punpckhwd xm2, xm0 ; 23 34 +.hv_w2_loop: + movq xm4, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + movhps xm4, [srcq+ssq*0] + pshufb xm4, xm5 + pmaddubsw xm4, xm6 + pmaddwd xm3, xm9, xm1 ; a0 b0 + mova xm1, xm2 + pmaddwd xm2, xm10 ; a1 b1 + phaddw xm4, xm4 + paddd xm3, xm2 + pmulhrsw xm4, xm7 + palignr xm2, xm4, xm0, 12 + mova xm0, xm4 + punpcklwd xm2, xm4 ; 45 56 + pmaddwd xm4, xm11, xm2 ; a2 b2 + paddd xm3, xm8 + paddd xm3, xm4 + psrad xm3, 10 + packssdw xm3, xm3 + packuswb xm3, xm3 + pextrw [dstq+dsq*0], xm3, 0 + pextrw [dstq+dsq*1], xm3, 1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .hv_w2_loop + RET +.hv_w4: + mova m5, [subpel_h_shuf4] + vpbroadcastq m2, [srcq+nsq*2] + vpbroadcastq m4, [srcq+nsq*1] + vpbroadcastq m1, [srcq+ssq*0] + vpbroadcastq m3, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + vpbroadcastq m0, [srcq+ssq*0] + vpblendd m2, m4, 0xcc ; 0 1 + vpblendd m1, m3, 0xcc ; 2 3 + pshufb m2, m5 + pshufb m1, m5 + pshufb m0, m5 + pmaddubsw m2, m6 + pmaddubsw m1, m6 + pmaddubsw m0, m6 + phaddw m2, m1 + phaddw m0, m0 + pmulhrsw m2, m7 + pmulhrsw m0, m7 + palignr m3, m0, m2, 4 + punpcklwd m1, m2, m3 ; 01 12 + punpckhwd m2, m3 ; 23 34 +.hv_w4_loop: + vpbroadcastq m4, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + pmaddwd m3, m9, m1 ; a0 b0 + mova m1, m2 + pmaddwd m2, m10 ; a1 b1 + paddd m3, m2 + vpbroadcastq m2, [srcq+ssq*0] + vpblendd m4, m2, 0xcc ; 5 6 + pshufb m4, m5 + pmaddubsw m4, m6 + phaddw m4, m4 + pmulhrsw m4, m7 + palignr m2, m4, m0, 12 + mova m0, m4 + punpcklwd m2, m4 ; 45 56 + pmaddwd m4, m11, m2 ; a2 b2 + paddd m3, m8 + paddd m3, m4 + psrad m3, 10 + vextracti128 xm4, m3, 1 + packssdw xm3, xm4 + packuswb xm3, xm3 + pshuflw xm3, xm3, q3120 + movd [dstq+dsq*0], xm3 + pextrd [dstq+dsq*1], xm3, 1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .hv_w4_loop + RET +.hv_w8: + shr mxd, 16 + sub srcq, 2 + lea mxq, [r8+mxq*8+subpel_filters+1-put_avx2] + WIN64_PUSH_XMM 16 + vpbroadcastw m10, [mxq+0] + vpbroadcastw m11, [mxq+2] + vpbroadcastw m12, [mxq+4] + movzx mxd, myb + shr myd, 16 + cmp hd, 6 + cmovs myd, mxd + vpbroadcastq m0, [r8+myq*8+subpel_filters+1-put_avx2] + lea r6d, [wq*8-64] + vbroadcasti128 m8, [z_filter_s+ 6] + punpcklbw m0, m0 + vbroadcasti128 m9, [z_filter_s+10] + psraw m0, 8 ; sign-extend + mov nsq, ssq + pshufd m13, m0, q0000 + neg nsq + pshufd m14, m0, q1111 + lea r6d, [hq+r6*4] + pshufd m15, m0, q2222 +.hv_w8_loop0: + vbroadcasti128 m7, [z_filter_s+2] + movu xm3, [srcq+nsq*2] + lea r4, [srcq+ssq*2] + movu xm4, [srcq+nsq*1] + vbroadcasti128 m0, [srcq+ssq*0] + mov r7, dstq + vinserti128 m4, [srcq+ssq*1], 1 ; 1 3 + vpblendd m3, m0, 0xf0 ; 0 2 + vinserti128 m0, [r4+ssq*0], 1 ; 2 4 + vpbroadcastd m5, [pw_8192] +%macro HV_H_6TAP_W8 6 ; src/dst, tmp[1-2], shuf[1-3] + pshufb %2, %1, %4 + pmaddubsw %2, m10 + pshufb %3, %1, %5 + pmaddubsw %3, m11 + pshufb %1, %6 + pmaddubsw %1, m12 + paddw %2, %3 + paddw %1, %2 +%endmacro + HV_H_6TAP_W8 m3, m1, m2, m7, m8, m9 + HV_H_6TAP_W8 m4, m1, m2, m7, m8, m9 + HV_H_6TAP_W8 m0, m1, m2, m7, m8, m9 + vpermq m3, m3, q3120 + vpermq m4, m4, q3120 + vpermq m0, m0, q3120 + pmulhrsw m3, m5 + pmulhrsw m4, m5 + pmulhrsw m0, m5 + punpcklwd m1, m3, m4 ; 01 + punpckhwd m3, m4 ; 23 + punpcklwd m2, m4, m0 ; 12 + punpckhwd m4, m0 ; 34 +.hv_w8_loop: + movu xm7, [r4+ssq*1] + lea r4, [r4+ssq*2] + vinserti128 m7, [r4+ssq*0], 1 ; 5 6 + pmaddwd m5, m13, m1 ; a0 + mova m1, m3 + pmaddwd m6, m13, m2 ; b0 + mova m2, m4 + pmaddwd m3, m14 ; a1 + pmaddwd m4, m14 ; b1 + paddd m5, m3 + vbroadcasti128 m3, [z_filter_s+2] + paddd m6, m4 + HV_H_6TAP_W8 m7, m3, m4, m3, m8, m9 + vpbroadcastd m3, [pw_8192] + vpbroadcastd m4, [pd_512] + pmulhrsw m7, m3 + paddd m5, m4 + paddd m6, m4 + mova m4, m0 + vpermq m0, m7, q3120 + shufpd m4, m0, 0x05 + punpcklwd m3, m4, m0 ; 45 + pmaddwd m7, m15, m3 ; a2 + punpckhwd m4, m0 ; 67 + paddd m5, m7 + pmaddwd m7, m15, m4 ; b2 + paddd m6, m7 + psrad m5, 10 + psrad m6, 10 + packssdw m5, m6 + vextracti128 xm6, m5, 1 + packuswb xm5, xm6 + pshufd xm5, xm5, q3120 + movq [r7+dsq*0], xm5 + movhps [r7+dsq*1], xm5 + lea r7, [r7+dsq*2] + sub hd, 2 + jg .hv_w8_loop + add srcq, 8 + add dstq, 8 + movzx hd, r6b + sub r6d, 1<<8 + jg .hv_w8_loop0 + RET + +PUT_8TAP_FN smooth_sharp, SMOOTH, SHARP, put_8tap_8bpc +PUT_8TAP_FN sharp_smooth, SHARP, SMOOTH, put_8tap_8bpc +PUT_8TAP_FN regular_sharp, REGULAR, SHARP, put_8tap_8bpc +PUT_8TAP_FN sharp_regular, SHARP, REGULAR, put_8tap_8bpc +PUT_8TAP_FN sharp, SHARP, SHARP + +cglobal put_8tap_8bpc, 4, 9, 0, dst, ds, src, ss, w, h, mx, my, ss3 + imul mxd, mxm, 0x010101 + add mxd, t0d ; 8tap_h, mx, 4tap_h + imul myd, mym, 0x010101 + add myd, t1d ; 8tap_v, my, 4tap_v + lea r8, [put_avx2] + movsxd wq, wm + movifnidn hd, hm + test mxd, 0xf00 + jnz .h + test myd, 0xf00 + jz mangle(private_prefix %+ _put_6tap_8bpc_avx2).put +.v: + WIN64_SPILL_XMM 12, 15 movzx mxd, myb shr myd, 16 cmp hd, 6 @@ -1765,19 +2175,19 @@ cglobal put_8tap_8bpc, 4, 9, 0, dst, ds, src, ss, w, h, mx, my, ss3 .v_w64: .v_w128: lea r6d, [wq*8-128] - mov r4, srcq - mov r7, dstq + WIN64_PUSH_XMM 15 lea r6d, [hq+r6*2] .v_w16_loop0: vbroadcasti128 m4, [srcq+ssq*0] vbroadcasti128 m5, [srcq+ssq*1] + lea r4, [srcq+ss3q] vbroadcasti128 m6, [srcq+ssq*2] - add srcq, ss3q - vbroadcasti128 m0, [srcq+ssq*0] - vbroadcasti128 m1, [srcq+ssq*1] - vbroadcasti128 m2, [srcq+ssq*2] - add srcq, ss3q - vbroadcasti128 m3, [srcq+ssq*0] + vbroadcasti128 m0, [r4+ssq*0] + mov r7, dstq + vbroadcasti128 m1, [r4+ssq*1] + vbroadcasti128 m2, [r4+ssq*2] + add r4, ss3q + vbroadcasti128 m3, [r4+ssq*0] shufpd m4, m0, 0x0c shufpd m5, m1, 0x0c punpcklbw m1, m4, m5 ; 01 @@ -1789,50 +2199,137 @@ cglobal put_8tap_8bpc, 4, 9, 0, dst, ds, src, ss, w, h, mx, my, ss3 punpcklbw m3, m6, m0 ; 23 punpckhbw m6, m0 ; 56 .v_w16_loop: - vbroadcasti128 m12, [srcq+ssq*1] - lea srcq, [srcq+ssq*2] - vbroadcasti128 m13, [srcq+ssq*0] - pmaddubsw m14, m1, m8 ; a0 - pmaddubsw m15, m2, m8 ; b0 + vbroadcasti128 m12, [r4+ssq*1] + lea r4, [r4+ssq*2] + pmaddubsw m13, m1, m8 ; a0 + pmaddubsw m14, m2, m8 ; b0 mova m1, m3 mova m2, m4 pmaddubsw m3, m9 ; a1 pmaddubsw m4, m9 ; b1 - paddw m14, m3 - paddw m15, m4 + paddw m13, m3 + paddw m14, m4 mova m3, m5 mova m4, m6 pmaddubsw m5, m10 ; a2 pmaddubsw m6, m10 ; b2 - paddw m14, m5 - paddw m15, m6 + paddw m13, m5 + vbroadcasti128 m5, [r4+ssq*0] + paddw m14, m6 shufpd m6, m0, m12, 0x0d - shufpd m0, m12, m13, 0x0c + shufpd m0, m12, m5, 0x0c punpcklbw m5, m6, m0 ; 67 punpckhbw m6, m0 ; 78 pmaddubsw m12, m5, m11 ; a3 - pmaddubsw m13, m6, m11 ; b3 + paddw m13, m12 + pmaddubsw m12, m6, m11 ; b3 paddw m14, m12 - paddw m15, m13 + pmulhrsw m13, m7 pmulhrsw m14, m7 - pmulhrsw m15, m7 - packuswb m14, m15 - vpermq m14, m14, q3120 - mova [dstq+dsq*0], xm14 - vextracti128 [dstq+dsq*1], m14, 1 - lea dstq, [dstq+dsq*2] + packuswb m13, m14 + vpermq m13, m13, q3120 + mova [r7+dsq*0], xm13 + vextracti128 [r7+dsq*1], m13, 1 + lea r7, [r7+dsq*2] sub hd, 2 jg .v_w16_loop - add r4, 16 - add r7, 16 + add srcq, 16 + add dstq, 16 movzx hd, r6b - mov srcq, r4 - mov dstq, r7 sub r6d, 1<<8 jg .v_w16_loop0 RET +.h: +.h_w2: +.h_w4: + test myd, 0xf00 + jnz .hv + vpbroadcastd m5, [pw_34] ; 2 + (8 << 2) + cmp wd, 4 + jle mangle(private_prefix %+ _put_6tap_8bpc_avx2).h_w2 + WIN64_SPILL_XMM 11 + tzcnt wd, wd + vbroadcasti128 m6, [subpel_h_shufA] + shr mxd, 16 + vbroadcasti128 m7, [subpel_h_shufB] + sub srcq, 3 + vbroadcasti128 m8, [subpel_h_shufC] + movzx wd, word [r8+wq*2+table_offset(put, _8tap_h)] + vpbroadcastd m9, [r8+mxq*8+subpel_filters-put_avx2+0] + vpbroadcastd m10, [r8+mxq*8+subpel_filters-put_avx2+4] + add wq, r8 + jmp wq +.h_w8: +%macro PUT_8TAP_H 4 ; dst/src, tmp[1-3] + pshufb m%2, m%1, m7 + pshufb m%3, m%1, m8 + pshufb m%1, m6 + pmaddubsw m%4, m%2, m9 + pmaddubsw m%2, m10 + pmaddubsw m%3, m10 + pmaddubsw m%1, m9 + paddw m%3, m%4 + paddw m%1, m%2 + phaddw m%1, m%3 + paddw m%1, m5 + psraw m%1, 6 +%endmacro + movu xm0, [srcq+ssq*0] + vinserti128 m0, [srcq+ssq*1], 1 + lea srcq, [srcq+ssq*2] + PUT_8TAP_H 0, 1, 2, 3 + vextracti128 xm1, m0, 1 + packuswb xm0, xm1 + movq [dstq+dsq*0], xm0 + movhps [dstq+dsq*1], xm0 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .h_w8 + RET +.h_w16: + movu xm0, [srcq+ssq*0+8*0] + vinserti128 m0, [srcq+ssq*1+8*0], 1 + movu xm1, [srcq+ssq*0+8*1] + vinserti128 m1, [srcq+ssq*1+8*1], 1 + PUT_8TAP_H 0, 2, 3, 4 + lea srcq, [srcq+ssq*2] + PUT_8TAP_H 1, 2, 3, 4 + packuswb m0, m1 + mova [dstq+dsq*0], xm0 + vextracti128 [dstq+dsq*1], m0, 1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .h_w16 + RET +.h_w32: + xor r6d, r6d + jmp .h_start +.h_w64: + mov r6, -32*1 + jmp .h_start +.h_w128: + mov r6, -32*3 +.h_start: + sub srcq, r6 + sub dstq, r6 + mov r4, r6 +.h_loop: + movu m0, [srcq+r6+8*0] + movu m1, [srcq+r6+8*1] + PUT_8TAP_H 0, 2, 3, 4 + PUT_8TAP_H 1, 2, 3, 4 + packuswb m0, m1 + mova [dstq+r6], m0 + add r6, 32 + jle .h_loop + add srcq, ssq + add dstq, dsq + mov r6, r4 + dec hd + jg .h_loop + RET .hv: - WIN64_SPILL_XMM 16 + WIN64_SPILL_XMM 14, 16 cmp wd, 4 jg .hv_w8 movzx mxd, mxb @@ -1975,6 +2472,7 @@ cglobal put_8tap_8bpc, 4, 9, 0, dst, ds, src, ss, w, h, mx, my, ss3 jg .hv_w4_loop RET .hv_w8: + WIN64_PUSH_XMM 16 shr mxd, 16 sub srcq, 3 vpbroadcastd m10, [r8+mxq*8+subpel_filters-put_avx2+0] @@ -1993,24 +2491,23 @@ cglobal put_8tap_8bpc, 4, 9, 0, dst, ds, src, ss, w, h, mx, my, ss3 pshufd m14, m0, q2222 pshufd m15, m0, q3333 lea r6d, [wq*8-64] - mov r4, srcq - mov r7, dstq lea r6d, [hq+r6*4] .hv_w8_loop0: vbroadcasti128 m7, [subpel_h_shufA] movu xm4, [srcq+ssq*0] + lea r4, [srcq+ss3q] vbroadcasti128 m8, [subpel_h_shufB] movu xm5, [srcq+ssq*1] + mov r7, dstq vbroadcasti128 m9, [subpel_h_shufC] movu xm6, [srcq+ssq*2] - add srcq, ss3q - vbroadcasti128 m0, [srcq+ssq*0] - vpblendd m4, m0, 0xf0 ; 0 3 - vinserti128 m5, [srcq+ssq*1], 1 ; 1 4 - vinserti128 m6, [srcq+ssq*2], 1 ; 2 5 - add srcq, ss3q - vinserti128 m0, [srcq+ssq*0], 1 ; 3 6 -%macro HV_H_W8 4-7 ; src/dst, tmp[1-3], shuf[1-3] + vbroadcasti128 m0, [r4+ssq*0] + vpblendd m4, m0, 0xf0 ; 0 3 + vinserti128 m5, [r4+ssq*1], 1 ; 1 4 + vinserti128 m6, [r4+ssq*2], 1 ; 2 5 + add r4, ss3q + vinserti128 m0, [r4+ssq*0], 1 ; 3 6 +%macro HV_H_8TAP_W8 4-7 ; src/dst, tmp[1-3], shuf[1-3] pshufb %3, %1, %6 pshufb %4, %1, %7 pshufb %1, %5 @@ -2022,10 +2519,10 @@ cglobal put_8tap_8bpc, 4, 9, 0, dst, ds, src, ss, w, h, mx, my, ss3 paddw %1, %3 phaddw %1, %2 %endmacro - HV_H_W8 m4, m1, m2, m3, m7, m8, m9 - HV_H_W8 m5, m1, m2, m3, m7, m8, m9 - HV_H_W8 m6, m1, m2, m3, m7, m8, m9 - HV_H_W8 m0, m1, m2, m3, m7, m8, m9 + HV_H_8TAP_W8 m4, m1, m2, m3, m7, m8, m9 + HV_H_8TAP_W8 m5, m1, m2, m3, m7, m8, m9 + HV_H_8TAP_W8 m6, m1, m2, m3, m7, m8, m9 + HV_H_8TAP_W8 m0, m1, m2, m3, m7, m8, m9 vpbroadcastd m7, [pw_8192] vpermq m4, m4, q3120 vpermq m5, m5, q3120 @@ -2043,9 +2540,9 @@ cglobal put_8tap_8bpc, 4, 9, 0, dst, ds, src, ss, w, h, mx, my, ss3 punpckhwd m6, m7 ; 56 .hv_w8_loop: vextracti128 r6m, m0, 1 ; not enough registers - movu xm0, [srcq+ssq*1] - lea srcq, [srcq+ssq*2] - vinserti128 m0, [srcq+ssq*0], 1 ; 7 8 + movu xm0, [r4+ssq*1] + lea r4, [r4+ssq*2] + vinserti128 m0, [r4+ssq*0], 1 ; 7 8 pmaddwd m8, m1, m12 ; a0 pmaddwd m9, m2, m12 ; b0 mova m1, m3 @@ -2063,15 +2560,15 @@ cglobal put_8tap_8bpc, 4, 9, 0, dst, ds, src, ss, w, h, mx, my, ss3 vbroadcasti128 m6, [subpel_h_shufB] vbroadcasti128 m7, [subpel_h_shufC] vbroadcasti128 m5, [subpel_h_shufA] - HV_H_W8 m0, m5, m6, m7, m5, m6, m7 + HV_H_8TAP_W8 m0, m5, m6, m7, m5, m6, m7 vpbroadcastd m5, [pw_8192] vpbroadcastd m7, [pd_512] vbroadcasti128 m6, r6m pmulhrsw m0, m5 paddd m8, m7 paddd m9, m7 - vpermq m7, m0, q3120 ; 7 8 - shufpd m6, m6, m7, 0x04 ; 6 7 + vpermq m7, m0, q3120 ; 7 8 + shufpd m6, m7, 0x04 ; 6 7 punpcklwd m5, m6, m7 ; 67 punpckhwd m6, m7 ; 78 pmaddwd m7, m5, m15 ; a3 @@ -2084,34 +2581,18 @@ cglobal put_8tap_8bpc, 4, 9, 0, dst, ds, src, ss, w, h, mx, my, ss3 vextracti128 xm7, m8, 1 packuswb xm8, xm7 pshufd xm7, xm8, q3120 - movq [dstq+dsq*0], xm7 - movhps [dstq+dsq*1], xm7 - lea dstq, [dstq+dsq*2] + movq [r7+dsq*0], xm7 + movhps [r7+dsq*1], xm7 + lea r7, [r7+dsq*2] sub hd, 2 jg .hv_w8_loop - add r4, 8 - add r7, 8 + add srcq, 8 + add dstq, 8 movzx hd, r6b - mov srcq, r4 - mov dstq, r7 sub r6d, 1<<8 jg .hv_w8_loop0 RET -%macro PREP_8TAP_H 0 - pshufb m1, m0, m5 - pshufb m2, m0, m6 - pshufb m3, m0, m7 - pmaddubsw m1, m8 - pmaddubsw m0, m2, m8 - pmaddubsw m2, m9 - pmaddubsw m3, m9 - paddw m1, m2 - paddw m0, m3 - phaddw m0, m1, m0 - pmulhrsw m0, m4 -%endmacro - %if WIN64 DECLARE_REG_TMP 6, 4 %else @@ -2119,71 +2600,197 @@ DECLARE_REG_TMP 6, 7 %endif %define PREP_8TAP_FN FN prep_8tap, -PREP_8TAP_FN sharp, SHARP, SHARP -PREP_8TAP_FN sharp_smooth, SHARP, SMOOTH -PREP_8TAP_FN smooth_sharp, SMOOTH, SHARP -PREP_8TAP_FN smooth, SMOOTH, SMOOTH -PREP_8TAP_FN sharp_regular, SHARP, REGULAR -PREP_8TAP_FN regular_sharp, REGULAR, SHARP -PREP_8TAP_FN smooth_regular, SMOOTH, REGULAR -PREP_8TAP_FN regular_smooth, REGULAR, SMOOTH +PREP_8TAP_FN smooth, SMOOTH, SMOOTH, prep_6tap_8bpc +PREP_8TAP_FN smooth_regular, SMOOTH, REGULAR, prep_6tap_8bpc +PREP_8TAP_FN regular_smooth, REGULAR, SMOOTH, prep_6tap_8bpc PREP_8TAP_FN regular, REGULAR, REGULAR -cglobal prep_8tap_8bpc, 3, 8, 0, tmp, src, stride, w, h, mx, my, stride3 +cglobal prep_6tap_8bpc, 3, 8, 0, tmp, src, ss, w, h, mx, my, ns imul mxd, mxm, 0x010101 - add mxd, t0d ; 8tap_h, mx, 4tap_h + add mxd, t0d ; 6tap_h, mx, 4tap_h imul myd, mym, 0x010101 - add myd, t1d ; 8tap_v, my, 4tap_v + add myd, t1d ; 6tap_v, my, 4tap_v lea r7, [prep%+SUFFIX] - movsxd wq, wm + mov wd, wm movifnidn hd, hm test mxd, 0xf00 jnz .h test myd, 0xf00 jnz .v +.prep: tzcnt wd, wd movzx wd, word [r7+wq*2+table_offset(prep,)] add wq, r7 - lea r6, [strideq*3] + lea r6, [ssq*3] %if WIN64 pop r7 %endif jmp wq -.h: - test myd, 0xf00 - jnz .hv - vpbroadcastd m4, [pw_8192] - vbroadcasti128 m5, [subpel_h_shufA] - WIN64_SPILL_XMM 10 - cmp wd, 4 - je .h_w4 - tzcnt wd, wd - vbroadcasti128 m6, [subpel_h_shufB] - vbroadcasti128 m7, [subpel_h_shufC] - shr mxd, 16 - sub srcq, 3 - movzx wd, word [r7+wq*2+table_offset(prep, _8tap_h)] - vpbroadcastd m8, [r7+mxq*8+subpel_filters-prep%+SUFFIX+0] - vpbroadcastd m9, [r7+mxq*8+subpel_filters-prep%+SUFFIX+4] - add wq, r7 - jmp wq +.v: + WIN64_SPILL_XMM 10, 12 + movzx mxd, myb + shr myd, 16 + cmp hd, 4 + cmove myd, mxd + lea myq, [r7+myq*8+subpel_filters+1-prep%+SUFFIX] + vpbroadcastd m9, [pw_8192] + vpbroadcastw m6, [myq+0] + mov nsq, ssq + vpbroadcastw m7, [myq+2] + neg nsq + vpbroadcastw m8, [myq+4] + cmp wd, 8 + jg .v_w16 + je .v_w8 +.v_w4: + movd xm2, [srcq+nsq*2] + pinsrd xm2, [srcq+nsq*1], 1 + vpbroadcastd m1, [srcq+ssq*0] + vpbroadcastd m3, [srcq+ssq*1] + vpbroadcastd m0, [srcq+ssq*2] + vbroadcasti128 m5, [deint_shuf4] + vpblendd m1, m2, 0xeb + punpcklqdq m3, m0 + vpblendd m1, m3, 0x60 ; 0 1 2 _ 2 3 4 _ + pshufb m1, m5 ; 01 12 23 34 +.v_w4_loop: + lea srcq, [srcq+ssq*4] + pinsrd xm0, [srcq+nsq*1], 1 + vpbroadcastd m2, [srcq+ssq*0] + vpbroadcastd m3, [srcq+ssq*1] + vpblendd m2, m0, 0xeb + vpbroadcastd m0, [srcq+ssq*2] + punpcklqdq m3, m0 + vpblendd m2, m3, 0x60 ; 4 5 6 _ 6 7 8 _ + pshufb m2, m5 ; 45 56 67 78 + pmaddubsw m3, m1, m6 ; a0 b0 c0 d0 + vperm2i128 m1, m2, 0x21 ; 23 34 45 56 + pmaddubsw m4, m2, m8 ; a2 b2 c2 d2 + pmaddubsw m1, m7 ; a1 b1 c1 d1 + paddw m3, m4 + paddw m3, m1 + pmulhrsw m3, m9 + mova m1, m2 + mova [tmpq], m3 + add tmpq, 32 + sub hd, 4 + jg .v_w4_loop + RET +.v_w8: + movq xm1, [srcq+nsq*2] + vpbroadcastq m3, [srcq+nsq*1] + vpbroadcastq m2, [srcq+ssq*0] + vpbroadcastq m4, [srcq+ssq*1] + vpbroadcastq m0, [srcq+ssq*2] + vpblendd m1, m3, 0x30 + vpblendd m3, m2, 0x30 + punpcklbw m1, m3 ; 01 12 + vpblendd m2, m4, 0x30 + vpblendd m4, m0, 0x30 + punpcklbw m2, m4 ; 23 34 +.v_w8_loop: + lea srcq, [srcq+ssq*4] + pmaddubsw m1, m6 ; a0 + vpbroadcastq m3, [srcq+nsq*1] + pmaddubsw m4, m2, m7 ; a1 + pmaddubsw m5, m2, m6 ; b0 + vpbroadcastq m2, [srcq+ssq*0] + vpblendd m0, m3, 0x30 + vpblendd m3, m2, 0x30 + paddw m4, m1 + punpcklbw m1, m0, m3 ; 45 56 + vpbroadcastq m3, [srcq+ssq*1] + vpbroadcastq m0, [srcq+ssq*2] + vpblendd m2, m3, 0x30 + vpblendd m3, m0, 0x30 + punpcklbw m2, m3 ; 67 78 + pmaddubsw m3, m1, m7 ; b1 + paddw m5, m3 + pmaddubsw m3, m1, m8 ; a2 + paddw m4, m3 + pmaddubsw m3, m2, m8 ; b2 + paddw m5, m3 + pmulhrsw m4, m9 + pmulhrsw m5, m9 + mova [tmpq+32*0], m4 + mova [tmpq+32*1], m5 + add tmpq, 32*2 + sub hd, 4 + jg .v_w8_loop + RET +.v_w16: + lea r6d, [wq*2-32] + lea srcq, [srcq+nsq*2] + WIN64_PUSH_XMM 12 + lea r6d, [hq+r6*8] +.v_w16_loop0: + vbroadcasti128 m3, [srcq+ssq*0] + lea r5, [srcq+ssq*2] + vbroadcasti128 m4, [srcq+ssq*1] + mov r7, tmpq + vbroadcasti128 m0, [r5+ssq*0] + vbroadcasti128 m1, [r5+ssq*1] + lea r5, [r5+ssq*2] + vbroadcasti128 m2, [r5+ssq*0] + shufpd m3, m0, 0x0c + shufpd m4, m1, 0x0c + punpcklbw m1, m3, m4 ; 01 + punpckhbw m3, m4 ; 23 + shufpd m0, m2, 0x0c + punpcklbw m2, m4, m0 ; 12 + punpckhbw m4, m0 ; 34 +.v_w16_loop: + vbroadcasti128 m5, [r5+ssq*1] + pmaddubsw m10, m1, m6 ; a0 + lea r5, [r5+ssq*2] + pmaddubsw m11, m2, m6 ; b0 + mova m1, m3 + pmaddubsw m3, m7 ; a1 + mova m2, m4 + pmaddubsw m4, m7 ; b1 + paddw m10, m3 + vbroadcasti128 m3, [r5+ssq*0] + paddw m11, m4 + shufpd m4, m0, m5, 0x0d + shufpd m0, m5, m3, 0x0c + punpcklbw m3, m4, m0 ; 45 + punpckhbw m4, m0 ; 56 + pmaddubsw m5, m3, m8 ; a2 + paddw m10, m5 + pmaddubsw m5, m4, m8 ; b2 + paddw m11, m5 + pmulhrsw m10, m9 + pmulhrsw m11, m9 + mova [r7+wq*0], m10 + mova [r7+wq*2], m11 + lea r7, [r7+wq*4] + sub hd, 2 + jg .v_w16_loop + add srcq, 16 + add tmpq, 32 + movzx hd, r6b + sub r6d, 1<<8 + jg .v_w16_loop0 + RET .h_w4: + RESET_STACK_STATE movzx mxd, mxb + vbroadcasti128 m3, [subpel_h_shufA] dec srcq - vpbroadcastd m6, [r7+mxq*8+subpel_filters-prep%+SUFFIX+2] - lea stride3q, [strideq*3] + vpbroadcastd m5, [r7+mxq*8+subpel_filters-prep%+SUFFIX+2] + lea r3, [ssq*3] .h_w4_loop: - movq xm0, [srcq+strideq*0] - vpbroadcastq m2, [srcq+strideq*2] - movq xm1, [srcq+strideq*1] - vpblendd m0, m2, 0xf0 - vpbroadcastq m2, [srcq+stride3q ] - lea srcq, [srcq+strideq*4] - vpblendd m1, m2, 0xf0 - pshufb m0, m5 - pshufb m1, m5 - pmaddubsw m0, m6 - pmaddubsw m1, m6 + movq xm0, [srcq+ssq*0] + vpbroadcastq m2, [srcq+ssq*2] + movq xm1, [srcq+ssq*1] + vpblendd m0, m2, 0x30 + vpbroadcastq m2, [srcq+r3 ] + lea srcq, [srcq+ssq*4] + vpblendd m1, m2, 0x30 + pshufb m0, m3 + pshufb m1, m3 + pmaddubsw m0, m5 + pmaddubsw m1, m5 phaddw m0, m1 pmulhrsw m0, m4 mova [tmpq], m0 @@ -2191,25 +2798,56 @@ cglobal prep_8tap_8bpc, 3, 8, 0, tmp, src, stride, w, h, mx, my, stride3 sub hd, 4 jg .h_w4_loop RET +.h: + test myd, 0xf00 + jnz .hv + vpbroadcastd m4, [pw_8192] + cmp wd, 4 + je .h_w4 + WIN64_SPILL_XMM 10 + tzcnt wd, wd + vbroadcasti128 m3, [z_filter_s+ 2] + shr mxd, 16 + vbroadcasti128 m5, [z_filter_s+ 6] + sub srcq, 2 + vbroadcasti128 m6, [z_filter_s+10] + lea mxq, [r7+mxq*8+subpel_filters+1-prep%+SUFFIX] + movzx wd, word [r7+wq*2+table_offset(prep, _6tap_h)] + vpbroadcastw m7, [mxq+0] + vpbroadcastw m8, [mxq+2] + add wq, r7 + vpbroadcastw m9, [mxq+4] + jmp wq .h_w8: - movu xm0, [srcq+strideq*0] - vinserti128 m0, [srcq+strideq*1], 1 - lea srcq, [srcq+strideq*2] - PREP_8TAP_H + movu xm0, [srcq+ssq*0] + vinserti128 m0, [srcq+ssq*1], 1 + lea srcq, [srcq+ssq*2] +%macro PREP_6TAP_H 0 + pshufb m1, m0, m3 + pmaddubsw m1, m7 + pshufb m2, m0, m5 + pmaddubsw m2, m8 + pshufb m0, m6 + pmaddubsw m0, m9 + paddw m1, m2 + paddw m0, m1 + pmulhrsw m0, m4 +%endmacro + PREP_6TAP_H mova [tmpq], m0 add tmpq, 32 sub hd, 2 jg .h_w8 RET .h_w16: - movu xm0, [srcq+strideq*0+8*0] - vinserti128 m0, [srcq+strideq*0+8*1], 1 - PREP_8TAP_H + movu xm0, [srcq+ssq*0+8*0] + vinserti128 m0, [srcq+ssq*0+8*1], 1 + PREP_6TAP_H mova [tmpq+32*0], m0 - movu xm0, [srcq+strideq*1+8*0] - vinserti128 m0, [srcq+strideq*1+8*1], 1 - lea srcq, [srcq+strideq*2] - PREP_8TAP_H + movu xm0, [srcq+ssq*1+8*0] + vinserti128 m0, [srcq+ssq*1+8*1], 1 + lea srcq, [srcq+ssq*2] + PREP_6TAP_H mova [tmpq+32*1], m0 add tmpq, 32*2 sub hd, 2 @@ -2229,27 +2867,219 @@ cglobal prep_8tap_8bpc, 3, 8, 0, tmp, src, stride, w, h, mx, my, stride3 .h_loop: movu xm0, [srcq+r6+8*0] vinserti128 m0, [srcq+r6+8*1], 1 - PREP_8TAP_H + PREP_6TAP_H mova [tmpq+32*0], m0 movu xm0, [srcq+r6+8*2] vinserti128 m0, [srcq+r6+8*3], 1 - PREP_8TAP_H + PREP_6TAP_H mova [tmpq+32*1], m0 add tmpq, 32*2 add r6, 32 jle .h_loop - add srcq, strideq + add srcq, ssq mov r6, r5 dec hd jg .h_loop RET +.hv: + WIN64_SPILL_XMM 14, 16 + cmp wd, 4 + jne .hv_w8 +.hv_w4: + movzx mxd, mxb + dec srcq + vpbroadcastd m7, [r7+mxq*8+subpel_filters-prep%+SUFFIX+2] + movzx mxd, myb + shr myd, 16 + cmp hd, 4 + cmove myd, mxd + mova m6, [subpel_h_shuf4] + vpbroadcastq m0, [r7+myq*8+subpel_filters+1-prep%+SUFFIX] + mov nsq, ssq + pmovzxbd m13, [deint_shuf4] + neg nsq + vpbroadcastd m8, [pw_8192] + vpbroadcastd m9, [pd_32] + punpcklbw m0, m0 + vpbroadcastq m2, [srcq+nsq*2] + psraw m0, 8 ; sign-extend + vpbroadcastq m4, [srcq+nsq*1] + pshufd m10, m0, q0000 + vpbroadcastq m1, [srcq+ssq*0] + pshufd m11, m0, q1111 + vpbroadcastq m3, [srcq+ssq*1] + pshufd m12, m0, q2222 + vpbroadcastq m0, [srcq+ssq*2] + vpblendd m2, m4, 0xcc ; 0 1 + vpblendd m1, m3, 0xcc ; 2 3 + pshufb m2, m6 + pshufb m1, m6 + pshufb m0, m6 + pmaddubsw m2, m7 + pmaddubsw m1, m7 + pmaddubsw m0, m7 + phaddw m2, m1 ; 0 1 2 3 + phaddw m0, m0 ; 4 + pmulhrsw m2, m8 + pmulhrsw m0, m8 + palignr m0, m2, 4 + punpcklwd m1, m2, m0 ; 01 12 + punpckhwd m2, m0 ; 23 34 +.hv_w4_loop: + pmaddwd m4, m10, m1 ; a0 b0 + lea srcq, [srcq+ssq*4] + pmaddwd m5, m2, m10 ; c0 d0 + vpbroadcastq m1, [srcq+nsq*1] + pmaddwd m2, m11 ; a1 b1 + vpbroadcastq m3, [srcq+ssq*0] + paddd m4, m2 + vpbroadcastq m2, [srcq+ssq*1] + vpblendd m1, m3, 0xcc ; 5 6 + vpbroadcastq m3, [srcq+ssq*2] + vpblendd m2, m3, 0xcc ; 7 8 + pshufb m1, m6 + pshufb m2, m6 + pmaddubsw m1, m7 + pmaddubsw m2, m7 + phaddw m1, m2 ; 5 6 7 8 + pmulhrsw m1, m8 + paddd m5, m9 + paddd m4, m9 + palignr m2, m1, m0, 12 + mova m0, m1 + punpcklwd m1, m2, m0 ; 45 56 + punpckhwd m2, m0 ; 67 78 + pmaddwd m3, m11, m1 ; c1 d1 + paddd m5, m3 + pmaddwd m3, m12, m1 ; a2 b2 + paddd m4, m3 + pmaddwd m3, m12, m2 ; c2 d2 + paddd m5, m3 + psrad m4, 6 + psrad m5, 6 + packssdw m4, m5 + vpermd m4, m13, m4 + mova [tmpq], m4 + add tmpq, 32 + sub hd, 4 + jg .hv_w4_loop + RET +.hv_w8: + shr mxd, 16 + lea mxq, [r7+mxq*8+subpel_filters+1-prep_avx2] + WIN64_PUSH_XMM 16 + vpbroadcastw m10, [mxq+0] + vpbroadcastw m11, [mxq+2] + vpbroadcastw m12, [mxq+4] + movzx mxd, myb + shr myd, 16 + cmp hd, 6 + cmovs myd, mxd + vpbroadcastq m0, [r7+myq*8+subpel_filters+1-prep_avx2] + lea r7, [ssq*2+2] + vbroadcasti128 m8, [z_filter_s+ 6] + punpcklbw m0, m0 + vbroadcasti128 m9, [z_filter_s+10] + psraw m0, 8 ; sign-extend + lea r6d, [wq*8-64] + pshufd m13, m0, q0000 + sub srcq, r7 + pshufd m14, m0, q1111 + lea r6d, [hq+r6*4] + pshufd m15, m0, q2222 +.hv_w8_loop0: + vbroadcasti128 m7, [z_filter_s+2] + movu xm3, [srcq+ssq*0] + lea r5, [srcq+ssq*2] + movu xm4, [srcq+ssq*1] + vbroadcasti128 m0, [r5+ssq*0] + mov r7, tmpq + vinserti128 m4, [r5+ssq*1], 1 ; 1 3 + lea r5, [r5+ssq*2] + vpblendd m3, m0, 0xf0 ; 0 2 + vinserti128 m0, [r5+ssq*0], 1 ; 2 4 + vpbroadcastd m5, [pw_8192] + HV_H_6TAP_W8 m3, m1, m2, m7, m8, m9 + HV_H_6TAP_W8 m4, m1, m2, m7, m8, m9 + HV_H_6TAP_W8 m0, m1, m2, m7, m8, m9 + vpermq m3, m3, q3120 + vpermq m4, m4, q3120 + vpermq m0, m0, q3120 + pmulhrsw m3, m5 + pmulhrsw m4, m5 + pmulhrsw m0, m5 + punpcklwd m1, m3, m4 ; 01 + punpckhwd m3, m4 ; 23 + punpcklwd m2, m4, m0 ; 12 + punpckhwd m4, m0 ; 34 +.hv_w8_loop: + movu xm7, [r5+ssq*1] + lea r5, [r5+ssq*2] + vinserti128 m7, [r5+ssq*0], 1 ; 5 6 + pmaddwd m5, m13, m1 ; a0 + mova m1, m3 + pmaddwd m6, m13, m2 ; b0 + mova m2, m4 + pmaddwd m3, m14 ; a1 + pmaddwd m4, m14 ; b1 + paddd m5, m3 + vbroadcasti128 m3, [z_filter_s+2] + paddd m6, m4 + HV_H_6TAP_W8 m7, m3, m4, m3, m8, m9 + vpbroadcastd m3, [pw_8192] + vpbroadcastd m4, [pd_32] + pmulhrsw m7, m3 + paddd m5, m4 + paddd m6, m4 + mova m4, m0 + vpermq m0, m7, q3120 + shufpd m4, m0, 0x05 + punpcklwd m3, m4, m0 ; 45 + pmaddwd m7, m15, m3 ; a2 + punpckhwd m4, m0 ; 67 + paddd m5, m7 + pmaddwd m7, m15, m4 ; b2 + paddd m6, m7 + psrad m5, 6 + psrad m6, 6 + packssdw m5, m6 + vpermq m5, m5, q3120 + mova [r7+wq*0], xm5 + vextracti128 [r7+wq*2], m5, 1 + lea r7, [r7+wq*4] + sub hd, 2 + jg .hv_w8_loop + add srcq, 8 + add tmpq, 16 + movzx hd, r6b + sub r6d, 1<<8 + jg .hv_w8_loop0 + RET + +PREP_8TAP_FN smooth_sharp, SMOOTH, SHARP, prep_8tap_8bpc +PREP_8TAP_FN sharp_smooth, SHARP, SMOOTH, prep_8tap_8bpc +PREP_8TAP_FN regular_sharp, REGULAR, SHARP, prep_8tap_8bpc +PREP_8TAP_FN sharp_regular, SHARP, REGULAR, prep_8tap_8bpc +PREP_8TAP_FN sharp, SHARP, SHARP + +cglobal prep_8tap_8bpc, 3, 8, 0, tmp, src, stride, w, h, mx, my, stride3 + imul mxd, mxm, 0x010101 + add mxd, t0d ; 8tap_h, mx, 4tap_h + imul myd, mym, 0x010101 + add myd, t1d ; 8tap_v, my, 4tap_v + lea r7, [prep%+SUFFIX] + mov wd, wm + movifnidn hd, hm + test mxd, 0xf00 + jnz .h + test myd, 0xf00 + jz mangle(private_prefix %+ _prep_6tap_8bpc_avx2).prep .v: - WIN64_SPILL_XMM 16 + WIN64_SPILL_XMM 12, 15 movzx mxd, myb ; Select 4-tap/8-tap filter multipliers. shr myd, 16 ; Note that the code is 8-tap only, having cmp hd, 4 ; a separate 4-tap code path for (4|8|16)x4 cmove myd, mxd ; had a negligible effect on performance. - ; TODO: Would a 6-tap code path be worth it? lea myq, [r7+myq*8+subpel_filters-prep%+SUFFIX] lea stride3q, [strideq*3] sub srcq, stride3q @@ -2359,72 +3189,154 @@ cglobal prep_8tap_8bpc, 3, 8, 0, tmp, src, stride, w, h, mx, my, stride3 jg .v_w8_loop RET .v_w16: - add wd, wd - mov r5, srcq - mov r7, tmpq - lea r6d, [hq+wq*8-256] + lea r6d, [wq*2-32] + WIN64_PUSH_XMM 15 + lea r6d, [hq+r6*8] .v_w16_loop0: vbroadcasti128 m4, [srcq+strideq*0] vbroadcasti128 m5, [srcq+strideq*1] - lea srcq, [srcq+strideq*2] - vbroadcasti128 m0, [srcq+strideq*1] - vbroadcasti128 m6, [srcq+strideq*0] - lea srcq, [srcq+strideq*2] - vbroadcasti128 m1, [srcq+strideq*0] - vbroadcasti128 m2, [srcq+strideq*1] - lea srcq, [srcq+strideq*2] - vbroadcasti128 m3, [srcq+strideq*0] - shufpd m4, m4, m0, 0x0c - shufpd m5, m5, m1, 0x0c + lea r5, [srcq+strideq*2] + vbroadcasti128 m0, [r5+strideq*1] + vbroadcasti128 m6, [r5+strideq*0] + lea r5, [r5+strideq*2] + vbroadcasti128 m1, [r5+strideq*0] + vbroadcasti128 m2, [r5+strideq*1] + lea r5, [r5+strideq*2] + vbroadcasti128 m3, [r5+strideq*0] + mov r7, tmpq + shufpd m4, m0, 0x0c + shufpd m5, m1, 0x0c punpcklbw m1, m4, m5 ; 01 punpckhbw m4, m5 ; 34 - shufpd m6, m6, m2, 0x0c + shufpd m6, m2, 0x0c punpcklbw m2, m5, m6 ; 12 punpckhbw m5, m6 ; 45 - shufpd m0, m0, m3, 0x0c + shufpd m0, m3, 0x0c punpcklbw m3, m6, m0 ; 23 punpckhbw m6, m0 ; 56 .v_w16_loop: - vbroadcasti128 m12, [srcq+strideq*1] - lea srcq, [srcq+strideq*2] - vbroadcasti128 m13, [srcq+strideq*0] - pmaddubsw m14, m1, m8 ; a0 - pmaddubsw m15, m2, m8 ; b0 + vbroadcasti128 m12, [r5+strideq*1] + lea r5, [r5+strideq*2] + pmaddubsw m13, m1, m8 ; a0 + pmaddubsw m14, m2, m8 ; b0 mova m1, m3 mova m2, m4 pmaddubsw m3, m9 ; a1 pmaddubsw m4, m9 ; b1 - paddw m14, m3 - paddw m15, m4 + paddw m13, m3 + paddw m14, m4 mova m3, m5 mova m4, m6 pmaddubsw m5, m10 ; a2 pmaddubsw m6, m10 ; b2 - paddw m14, m5 - paddw m15, m6 + paddw m13, m5 + vbroadcasti128 m5, [r5+strideq*0] + paddw m14, m6 shufpd m6, m0, m12, 0x0d - shufpd m0, m12, m13, 0x0c + shufpd m0, m12, m5, 0x0c punpcklbw m5, m6, m0 ; 67 punpckhbw m6, m0 ; 78 pmaddubsw m12, m5, m11 ; a3 - pmaddubsw m13, m6, m11 ; b3 + paddw m13, m12 + pmaddubsw m12, m6, m11 ; b3 paddw m14, m12 - paddw m15, m13 + pmulhrsw m13, m7 pmulhrsw m14, m7 - pmulhrsw m15, m7 - mova [tmpq+wq*0], m14 - mova [tmpq+wq*1], m15 - lea tmpq, [tmpq+wq*2] + mova [r7+wq*0], m13 + mova [r7+wq*2], m14 + lea r7, [r7+wq*4] sub hd, 2 jg .v_w16_loop - add r5, 16 - add r7, 32 + add srcq, 16 + add tmpq, 32 movzx hd, r6b - mov srcq, r5 - mov tmpq, r7 sub r6d, 1<<8 jg .v_w16_loop0 RET +.h: +.h_w4: + test myd, 0xf00 + jnz .hv + vpbroadcastd m4, [pw_8192] + cmp wd, 4 + je mangle(private_prefix %+ _prep_6tap_8bpc_avx2).h_w4 + WIN64_SPILL_XMM 10 + vbroadcasti128 m5, [subpel_h_shufA] + tzcnt wd, wd + vbroadcasti128 m6, [subpel_h_shufB] + vbroadcasti128 m7, [subpel_h_shufC] + shr mxd, 16 + sub srcq, 3 + movzx wd, word [r7+wq*2+table_offset(prep, _8tap_h)] + vpbroadcastd m8, [r7+mxq*8+subpel_filters-prep%+SUFFIX+0] + vpbroadcastd m9, [r7+mxq*8+subpel_filters-prep%+SUFFIX+4] + add wq, r7 + jmp wq +.h_w8: + movu xm0, [srcq+strideq*0] + vinserti128 m0, [srcq+strideq*1], 1 + lea srcq, [srcq+strideq*2] +%macro PREP_8TAP_H 0 + pshufb m1, m0, m5 + pshufb m2, m0, m6 + pshufb m3, m0, m7 + pmaddubsw m1, m8 + pmaddubsw m0, m2, m8 + pmaddubsw m2, m9 + pmaddubsw m3, m9 + paddw m1, m2 + paddw m0, m3 + phaddw m0, m1, m0 + pmulhrsw m0, m4 +%endmacro + PREP_8TAP_H + mova [tmpq], m0 + add tmpq, 32 + sub hd, 2 + jg .h_w8 + RET +.h_w16: + movu xm0, [srcq+strideq*0+8*0] + vinserti128 m0, [srcq+strideq*0+8*1], 1 + PREP_8TAP_H + mova [tmpq+32*0], m0 + movu xm0, [srcq+strideq*1+8*0] + vinserti128 m0, [srcq+strideq*1+8*1], 1 + lea srcq, [srcq+strideq*2] + PREP_8TAP_H + mova [tmpq+32*1], m0 + add tmpq, 32*2 + sub hd, 2 + jg .h_w16 + RET +.h_w32: + xor r6d, r6d + jmp .h_start +.h_w64: + mov r6, -32*1 + jmp .h_start +.h_w128: + mov r6, -32*3 +.h_start: + sub srcq, r6 + mov r5, r6 +.h_loop: + movu xm0, [srcq+r6+8*0] + vinserti128 m0, [srcq+r6+8*1], 1 + PREP_8TAP_H + mova [tmpq+32*0], m0 + movu xm0, [srcq+r6+8*2] + vinserti128 m0, [srcq+r6+8*3], 1 + PREP_8TAP_H + mova [tmpq+32*1], m0 + add tmpq, 32*2 + add r6, 32 + jle .h_loop + add srcq, strideq + mov r6, r5 + dec hd + jg .h_loop + RET .hv: WIN64_SPILL_XMM 16 cmp wd, 4 @@ -2542,28 +3454,27 @@ cglobal prep_8tap_8bpc, 3, 8, 0, tmp, src, stride, w, h, mx, my, stride3 RET .hv_w8: lea r6d, [wq*8-64] - mov r5, srcq - mov r7, tmpq lea r6d, [hq+r6*4] .hv_w8_loop0: vbroadcasti128 m7, [subpel_h_shufA] movu xm4, [srcq+strideq*0] + lea r5, [srcq+strideq*2] vbroadcasti128 m8, [subpel_h_shufB] movu xm5, [srcq+strideq*1] - lea srcq, [srcq+strideq*2] + mov r7, tmpq vbroadcasti128 m9, [subpel_h_shufC] - movu xm6, [srcq+strideq*0] - vbroadcasti128 m0, [srcq+strideq*1] - lea srcq, [srcq+strideq*2] - vpblendd m4, m0, 0xf0 ; 0 3 - vinserti128 m5, [srcq+strideq*0], 1 ; 1 4 - vinserti128 m6, [srcq+strideq*1], 1 ; 2 5 - lea srcq, [srcq+strideq*2] - vinserti128 m0, [srcq+strideq*0], 1 ; 3 6 - HV_H_W8 m4, m1, m2, m3, m7, m8, m9 - HV_H_W8 m5, m1, m2, m3, m7, m8, m9 - HV_H_W8 m6, m1, m2, m3, m7, m8, m9 - HV_H_W8 m0, m1, m2, m3, m7, m8, m9 + movu xm6, [r5+strideq*0] + vbroadcasti128 m0, [r5+strideq*1] + lea r5, [r5+strideq*2] + vpblendd m4, m0, 0xf0 ; 0 3 + vinserti128 m5, [r5+strideq*0], 1 ; 1 4 + vinserti128 m6, [r5+strideq*1], 1 ; 2 5 + lea r5, [r5+strideq*2] + vinserti128 m0, [r5+strideq*0], 1 ; 3 6 + HV_H_8TAP_W8 m4, m1, m2, m3, m7, m8, m9 + HV_H_8TAP_W8 m5, m1, m2, m3, m7, m8, m9 + HV_H_8TAP_W8 m6, m1, m2, m3, m7, m8, m9 + HV_H_8TAP_W8 m0, m1, m2, m3, m7, m8, m9 vpbroadcastd m7, [pw_8192] vpermq m4, m4, q3120 vpermq m5, m5, q3120 @@ -2580,10 +3491,10 @@ cglobal prep_8tap_8bpc, 3, 8, 0, tmp, src, stride, w, h, mx, my, stride3 punpcklwd m3, m6, m7 ; 23 punpckhwd m6, m7 ; 56 .hv_w8_loop: - vextracti128 [tmpq], m0, 1 ; not enough registers - movu xm0, [srcq+strideq*1] - lea srcq, [srcq+strideq*2] - vinserti128 m0, [srcq+strideq*0], 1 ; 7 8 + vextracti128 [r7], m0, 1 ; not enough registers + movu xm0, [r5+strideq*1] + lea r5, [r5+strideq*2] + vinserti128 m0, [r5+strideq*0], 1 ; 7 8 pmaddwd m8, m1, m12 ; a0 pmaddwd m9, m2, m12 ; b0 mova m1, m3 @@ -2601,15 +3512,15 @@ cglobal prep_8tap_8bpc, 3, 8, 0, tmp, src, stride, w, h, mx, my, stride3 vbroadcasti128 m6, [subpel_h_shufB] vbroadcasti128 m7, [subpel_h_shufC] vbroadcasti128 m5, [subpel_h_shufA] - HV_H_W8 m0, m5, m6, m7, m5, m6, m7 + HV_H_8TAP_W8 m0, m5, m6, m7, m5, m6, m7 vpbroadcastd m5, [pw_8192] vpbroadcastd m7, [pd_32] - vbroadcasti128 m6, [tmpq] + vbroadcasti128 m6, [r7] pmulhrsw m0, m5 paddd m8, m7 paddd m9, m7 - vpermq m7, m0, q3120 ; 7 8 - shufpd m6, m6, m7, 0x04 ; 6 7 + vpermq m7, m0, q3120 ; 7 8 + shufpd m6, m7, 0x04 ; 6 7 punpcklwd m5, m6, m7 ; 67 punpckhwd m6, m7 ; 78 pmaddwd m7, m5, m15 ; a3 @@ -2620,16 +3531,14 @@ cglobal prep_8tap_8bpc, 3, 8, 0, tmp, src, stride, w, h, mx, my, stride3 psrad m7, 6 packssdw m8, m7 vpermq m7, m8, q3120 - mova [tmpq+wq*0], xm7 - vextracti128 [tmpq+wq*2], m7, 1 - lea tmpq, [tmpq+wq*4] + mova [r7+wq*0], xm7 + vextracti128 [r7+wq*2], m7, 1 + lea r7, [r7+wq*4] sub hd, 2 jg .hv_w8_loop - add r5, 8 - add r7, 16 + add srcq, 8 + add tmpq, 16 movzx hd, r6b - mov srcq, r5 - mov tmpq, r7 sub r6d, 1<<8 jg .hv_w8_loop0 RET @@ -4008,14 +4917,14 @@ DECLARE_REG_TMP 6, 8 %define PREP_8TAP_SCALED_FN FN prep_8tap_scaled, BILIN_SCALED_FN put -PUT_8TAP_SCALED_FN sharp, SHARP, SHARP -PUT_8TAP_SCALED_FN sharp_smooth, SHARP, SMOOTH -PUT_8TAP_SCALED_FN smooth_sharp, SMOOTH, SHARP -PUT_8TAP_SCALED_FN smooth, SMOOTH, SMOOTH -PUT_8TAP_SCALED_FN sharp_regular, SHARP, REGULAR -PUT_8TAP_SCALED_FN regular_sharp, REGULAR, SHARP -PUT_8TAP_SCALED_FN smooth_regular, SMOOTH, REGULAR -PUT_8TAP_SCALED_FN regular_smooth, REGULAR, SMOOTH +PUT_8TAP_SCALED_FN sharp, SHARP, SHARP, put_8tap_scaled_8bpc +PUT_8TAP_SCALED_FN sharp_smooth, SHARP, SMOOTH, put_8tap_scaled_8bpc +PUT_8TAP_SCALED_FN smooth_sharp, SMOOTH, SHARP, put_8tap_scaled_8bpc +PUT_8TAP_SCALED_FN smooth, SMOOTH, SMOOTH, put_8tap_scaled_8bpc +PUT_8TAP_SCALED_FN sharp_regular, SHARP, REGULAR, put_8tap_scaled_8bpc +PUT_8TAP_SCALED_FN regular_sharp, REGULAR, SHARP, put_8tap_scaled_8bpc +PUT_8TAP_SCALED_FN smooth_regular, SMOOTH, REGULAR, put_8tap_scaled_8bpc +PUT_8TAP_SCALED_FN regular_smooth, REGULAR, SMOOTH, put_8tap_scaled_8bpc PUT_8TAP_SCALED_FN regular, REGULAR, REGULAR MC_8TAP_SCALED put @@ -4026,14 +4935,14 @@ DECLARE_REG_TMP 6, 7 %endif BILIN_SCALED_FN prep -PREP_8TAP_SCALED_FN sharp, SHARP, SHARP -PREP_8TAP_SCALED_FN sharp_smooth, SHARP, SMOOTH -PREP_8TAP_SCALED_FN smooth_sharp, SMOOTH, SHARP -PREP_8TAP_SCALED_FN smooth, SMOOTH, SMOOTH -PREP_8TAP_SCALED_FN sharp_regular, SHARP, REGULAR -PREP_8TAP_SCALED_FN regular_sharp, REGULAR, SHARP -PREP_8TAP_SCALED_FN smooth_regular, SMOOTH, REGULAR -PREP_8TAP_SCALED_FN regular_smooth, REGULAR, SMOOTH +PREP_8TAP_SCALED_FN sharp, SHARP, SHARP, prep_8tap_scaled_8bpc +PREP_8TAP_SCALED_FN sharp_smooth, SHARP, SMOOTH, prep_8tap_scaled_8bpc +PREP_8TAP_SCALED_FN smooth_sharp, SMOOTH, SHARP, prep_8tap_scaled_8bpc +PREP_8TAP_SCALED_FN smooth, SMOOTH, SMOOTH, prep_8tap_scaled_8bpc +PREP_8TAP_SCALED_FN sharp_regular, SHARP, REGULAR, prep_8tap_scaled_8bpc +PREP_8TAP_SCALED_FN regular_sharp, REGULAR, SHARP, prep_8tap_scaled_8bpc +PREP_8TAP_SCALED_FN smooth_regular, SMOOTH, REGULAR, prep_8tap_scaled_8bpc +PREP_8TAP_SCALED_FN regular_smooth, REGULAR, SMOOTH, prep_8tap_scaled_8bpc PREP_8TAP_SCALED_FN regular, REGULAR, REGULAR MC_8TAP_SCALED prep diff --git a/third_party/dav1d/src/x86/mc_avx512.asm b/third_party/dav1d/src/x86/mc_avx512.asm index f9043f1ad3..50e670ec25 100644 --- a/third_party/dav1d/src/x86/mc_avx512.asm +++ b/third_party/dav1d/src/x86/mc_avx512.asm @@ -89,55 +89,47 @@ wm_444_mask: db 1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 3 db 33, 35, 37, 39, 41, 43, 45, 47, 49, 51, 53, 55, 57, 59, 61, 63 db 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30 db 32, 34, 36, 38, 40, 42, 44, 46, 48, 50, 52, 54, 56, 58, 60, 62 -bilin_h_perm16: db 1, 0, 2, 1, 3, 2, 4, 3, 5, 4, 6, 5, 7, 6, 8, 7 - db 9, 8, 10, 9, 11, 10, 12, 11, 13, 12, 14, 13, 15, 14, 16, 15 - db 33, 32, 34, 33, 35, 34, 36, 35, 37, 36, 38, 37, 39, 38, 40, 39 - db 41, 40, 42, 41, 43, 42, 44, 43, 45, 44, 46, 45, 47, 46, 48, 47 -bilin_h_perm32: db 1, 0, 2, 1, 3, 2, 4, 3, 5, 4, 6, 5, 7, 6, 8, 7 - db 9, 8, 10, 9, 11, 10, 12, 11, 13, 12, 14, 13, 15, 14, 16, 15 - db 17, 16, 18, 17, 19, 18, 20, 19, 21, 20, 22, 21, 23, 22, 24, 23 - db 25, 24, 26, 25, 27, 26, 28, 27, 29, 28, 30, 29, 31, 30, 32, 31 -bilin_v_perm8: db 16, 0, 17, 1, 18, 2, 19, 3, 20, 4, 21, 5, 22, 6, 23, 7 - db 80, 16, 81, 17, 82, 18, 83, 19, 84, 20, 85, 21, 86, 22, 87, 23 - db 32, 80, 33, 81, 34, 82, 35, 83, 36, 84, 37, 85, 38, 86, 39, 87 - db 64, 32, 65, 33, 66, 34, 67, 35, 68, 36, 69, 37, 70, 38, 71, 39 -bilin_v_perm16: db 16, 0, 17, 1, 18, 2, 19, 3, 20, 4, 21, 5, 22, 6, 23, 7 - db 24, 8, 25, 9, 26, 10, 27, 11, 28, 12, 29, 13, 30, 14, 31, 15 - db 64, 16, 65, 17, 66, 18, 67, 19, 68, 20, 69, 21, 70, 22, 71, 23 - db 72, 24, 73, 25, 74, 26, 75, 27, 76, 28, 77, 29, 78, 30, 79, 31 -bilin_v_perm32: db 64, 0, 65, 1, 66, 2, 67, 3, 68, 4, 69, 5, 70, 6, 71, 7 - db 72, 8, 73, 9, 74, 10, 75, 11, 76, 12, 77, 13, 78, 14, 79, 15 - db 80, 16, 81, 17, 82, 18, 83, 19, 84, 20, 85, 21, 86, 22, 87, 23 - db 88, 24, 89, 25, 90, 26, 91, 27, 92, 28, 93, 29, 94, 30, 95, 31 -bilin_v_perm64: dq 0, 4, 1, 5, 2, 6, 3, 7 -spel_h_perm16a: db 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6 +bilin_h_perm16: db 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8 + db 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, 14, 14, 15, 15, 16 + db 32, 33, 33, 34, 34, 35, 35, 36, 36, 37, 37, 38, 38, 39, 39, 40 + db 40, 41, 41, 42, 42, 43, 43, 44, 44, 45, 45, 46, 46, 47, 47, 48 +bilin_h_perm32: db 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8 + db 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, 14, 14, 15, 15, 16 + db 16, 17, 17, 18, 18, 19, 19, 20, 20, 21, 21, 22, 22, 23, 23, 24 + db 24, 25, 25, 26, 26, 27, 27, 28, 28, 29, 29, 30, 30, 31, 31, 32 +bilin_v_perm8: db 0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23 + db 16, 80, 17, 81, 18, 82, 19, 83, 20, 84, 21, 85, 22, 86, 23, 87 + db 80, 32, 81, 33, 82, 34, 83, 35, 84, 36, 85, 37, 86, 38, 87, 39 + db 32, 64, 33, 65, 34, 66, 35, 67, 36, 68, 37, 69, 38, 70, 39, 71 +bilin_v_perm16: db 0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23 + db 8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31 + db 16, 64, 17, 65, 18, 66, 19, 67, 20, 68, 21, 69, 22, 70, 23, 71 + db 24, 72, 25, 73, 26, 74, 27, 75, 28, 76, 29, 77, 30, 78, 31, 79 +bilin_v_perm32: db 0, 64, 1, 65, 2, 66, 3, 67, 4, 68, 5, 69, 6, 70, 7, 71 + db 8, 72, 9, 73, 10, 74, 11, 75, 12, 76, 13, 77, 14, 78, 15, 79 + db 16, 80, 17, 81, 18, 82, 19, 83, 20, 84, 21, 85, 22, 86, 23, 87 + db 24, 88, 25, 89, 26, 90, 27, 91, 28, 92, 29, 93, 30, 94, 31, 95 +bilin_v_perm64: dd 0, 0, 4, 8, 1, 1, 5, 9, 2, 2, 6, 10, 3, 3, 7, 11 +spel_h_perm16: db 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6 db 8, 9, 10, 11, 9, 10, 11, 12, 10, 11, 12, 13, 11, 12, 13, 14 db 32, 33, 34, 35, 33, 34, 35, 36, 34, 35, 36, 37, 35, 36, 37, 38 db 40, 41, 42, 43, 41, 42, 43, 44, 42, 43, 44, 45, 43, 44, 45, 46 -spel_h_perm16b: db 4, 5, 6, 7, 5, 6, 7, 8, 6, 7, 8, 9, 7, 8, 9, 10 - db 12, 13, 14, 15, 13, 14, 15, 16, 14, 15, 16, 17, 15, 16, 17, 18 - db 36, 37, 38, 39, 37, 38, 39, 40, 38, 39, 40, 41, 39, 40, 41, 42 - db 44, 45, 46, 47, 45, 46, 47, 48, 46, 47, 48, 49, 47, 48, 49, 50 -spel_h_perm16c: db 8, 9, 10, 11, 9, 10, 11, 12, 10, 11, 12, 13, 11, 12, 13, 14 - db 16, 17, 18, 19, 17, 18, 19, 20, 18, 19, 20, 21, 19, 20, 21, 22 - db 40, 41, 42, 43, 41, 42, 43, 44, 42, 43, 44, 45, 43, 44, 45, 46 - db 48, 49, 50, 51, 49, 50, 51, 52, 50, 51, 52, 53, 51, 52, 53, 54 -spel_h_perm32a: db 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6 +spel_h_perm32: db 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6 db 8, 9, 10, 11, 9, 10, 11, 12, 10, 11, 12, 13, 11, 12, 13, 14 db 16, 17, 18, 19, 17, 18, 19, 20, 18, 19, 20, 21, 19, 20, 21, 22 db 24, 25, 26, 27, 25, 26, 27, 28, 26, 27, 28, 29, 27, 28, 29, 30 -spel_h_perm32b: db 4, 5, 6, 7, 5, 6, 7, 8, 6, 7, 8, 9, 7, 8, 9, 10 - db 12, 13, 14, 15, 13, 14, 15, 16, 14, 15, 16, 17, 15, 16, 17, 18 - db 20, 21, 22, 23, 21, 22, 23, 24, 22, 23, 24, 25, 23, 24, 25, 26 - db 28, 29, 30, 31, 29, 30, 31, 32, 30, 31, 32, 33, 31, 32, 33, 34 -spel_h_perm32c: db 8, 9, 10, 11, 9, 10, 11, 12, 10, 11, 12, 13, 11, 12, 13, 14 - db 16, 17, 18, 19, 17, 18, 19, 20, 18, 19, 20, 21, 19, 20, 21, 22 - db 24, 25, 26, 27, 25, 26, 27, 28, 26, 27, 28, 29, 27, 28, 29, 30 - db 32, 33, 34, 35, 33, 34, 35, 36, 34, 35, 36, 37, 35, 36, 37, 38 -spel_v_perm16: db 32, 0, 33, 1, 34, 2, 35, 3, 36, 4, 37, 5, 38, 6, 39, 7 +spel_v_perm8: db 0, 8, 1, 9, 2, 10, 3, 11, 4, 12, 5, 13, 6, 14, 7, 15 + db 8, 16, 9, 17, 10, 18, 11, 19, 12, 20, 13, 21, 14, 22, 15, 23 + db 16, 24, 17, 25, 18, 26, 19, 27, 20, 28, 21, 29, 22, 30, 23, 31 + db 24, 32, 25, 33, 26, 34, 27, 35, 28, 36, 29, 37, 30, 38, 31, 39 +spel_v_perm16a: db 32, 0, 33, 1, 34, 2, 35, 3, 36, 4, 37, 5, 38, 6, 39, 7 db 0, 8, 1, 9, 2, 10, 3, 11, 4, 12, 5, 13, 6, 14, 7, 15 db 40, 16, 41, 17, 42, 18, 43, 19, 44, 20, 45, 21, 46, 22, 47, 23 db 16, 24, 17, 25, 18, 26, 19, 27, 20, 28, 21, 29, 22, 30, 23, 31 +spel_v_perm16b: db 32, 0, 33, 1, 34, 2, 35, 3, 36, 4, 37, 5, 38, 6, 39, 7 + db 40, 16, 41, 17, 42, 18, 43, 19, 44, 20, 45, 21, 46, 22, 47, 23 + db 0, 8, 1, 9, 2, 10, 3, 11, 4, 12, 5, 13, 6, 14, 7, 15 + db 16, 24, 17, 25, 18, 26, 19, 27, 20, 28, 21, 29, 22, 30, 23, 31 spel_v_perm32: db 0, 32, 1, 33, 2, 34, 3, 35, 4, 36, 5, 37, 6, 38, 7, 39 db 8, 40, 9, 41, 10, 42, 11, 43, 12, 44, 13, 45, 14, 46, 15, 47 db 16, 48, 17, 49, 18, 50, 19, 51, 20, 52, 21, 53, 22, 54, 23, 55 @@ -154,34 +146,20 @@ spel_hv_perm8a: db 0, 1, 16, 17, 2, 3, 18, 19, 4, 5, 20, 21, 6, 7, 22, 2 db 8, 9, 24, 25, 10, 11, 26, 27, 12, 13, 28, 29, 14, 15, 30, 31 db 16, 17, 32, 33, 18, 19, 34, 35, 20, 21, 36, 37, 22, 23, 38, 39 db 24, 25, 40, 41, 26, 27, 42, 43, 28, 29, 44, 45, 30, 31, 46, 47 -spel_hv_perm8b: db 32, 33, 48, 49, 34, 35, 50, 51, 36, 37, 52, 53, 38, 39, 54, 55 - db 40, 41, 56, 57, 42, 43, 58, 59, 44, 45, 60, 61, 46, 47, 62, 63 - db 48, 49, 64, 65, 50, 51, 66, 67, 52, 53, 68, 69, 54, 55, 70, 71 - db 56, 57, 72, 73, 58, 59, 74, 75, 60, 61, 76, 77, 62, 63, 78, 79 -spel_hv_perm8c: db 34, 35, 0, 1, 38, 39, 4, 5, 42, 43, 8, 9, 46, 47, 12, 13 +spel_hv_perm8b: db 34, 35, 0, 1, 38, 39, 4, 5, 42, 43, 8, 9, 46, 47, 12, 13 db 50, 51, 16, 17, 54, 55, 20, 21, 58, 59, 24, 25, 62, 63, 28, 29 db 0, 1, 32, 33, 4, 5, 36, 37, 8, 9, 40, 41, 12, 13, 44, 45 db 16, 17, 48, 49, 20, 21, 52, 53, 24, 25, 56, 57, 28, 29, 60, 61 -spel_hv_end16: db 1, 3, 17, 19, 5, 7, 21, 23, 33, 35, 49, 51, 37, 39, 53, 55 - db 9, 11, 25, 27, 13, 15, 29, 31, 41, 43, 57, 59, 45, 47, 61, 63 spel_hv_perm16a:db 0, 1, 2, 3, 32, 33, 34, 35, 1, 2, 3, 4, 33, 34, 35, 36 db 2, 3, 4, 5, 34, 35, 36, 37, 3, 4, 5, 6, 35, 36, 37, 38 -spel_hv_perm16c:db 8, 9, 10, 11, 40, 41, 42, 43, 9, 10, 11, 12, 41, 42, 43, 44 + db 8, 9, 10, 11, 40, 41, 42, 43, 9, 10, 11, 12, 41, 42, 43, 44 db 10, 11, 12, 13, 42, 43, 44, 45, 11, 12, 13, 14, 43, 44, 45, 46 - db 16, 17, 18, 19, 48, 49, 50, 51, 17, 18, 19, 20, 49, 50, 51, 52 - db 18, 19, 20, 21, 50, 51, 52, 53, 19, 20, 21, 22, 51, 52, 53, 54 -spel_hv_perm16b:db 4, 5, 6, 7, 36, 37, 38, 39, 5, 6, 7, 8, 37, 38, 39, 40 - db 6, 7, 8, 9, 38, 39, 40, 41, 7, 8, 9, 10, 39, 40, 41, 42 - db 12, 13, 14, 15, 44, 45, 46, 47, 13, 14, 15, 16, 45, 46, 47, 48 - db 14, 15, 16, 17, 46, 47, 48, 49, 15, 16, 17, 18, 47, 48, 49, 50 -spel_hv_perm16d:db 0, 1, 2, 3, 1, 2, 3, 4, 4, 5, 6, 7, 5, 6, 7, 8 +spel_hv_perm16b:db 0, 1, 2, 3, 1, 2, 3, 4, 4, 5, 6, 7, 5, 6, 7, 8 db 2, 3, 4, 5, 3, 4, 5, 6, 6, 7, 8, 9, 7, 8, 9, 10 db 8, 9, 10, 11, 9, 10, 11, 12, 12, 13, 14, 15, 13, 14, 15, 16 db 10, 11, 12, 13, 11, 12, 13, 14, 14, 15, 16, 17, 15, 16, 17, 18 -spel_hv_perm16e:db 4, 5, 6, 7, 5, 6, 7, 8, 8, 9, 10, 11, 9, 10, 11, 12 - db 6, 7, 8, 9, 7, 8, 9, 10, 10, 11, 12, 13, 11, 12, 13, 14 - db 12, 13, 14, 15, 13, 14, 15, 16, 16, 17, 18, 19, 17, 18, 19, 20 - db 14, 15, 16, 17, 15, 16, 17, 18, 18, 19, 20, 21, 19, 20, 21, 22 +spel_hv_end16: db 1, 3, 17, 19, 5, 7, 21, 23, 33, 35, 49, 51, 37, 39, 53, 55 + db 9, 11, 25, 27, 13, 15, 29, 31, 41, 43, 57, 59, 45, 47, 61, 63 spel_hv_end: db 1, 3, 5, 7, 17, 19, 21, 23, 33, 35, 37, 39, 49, 51, 53, 55 deint_shuf4: db 0, 4, 1, 5, 2, 6, 3, 7, 4, 8, 5, 9, 6, 10, 7, 11 subpel_h_shuf4: db 0, 1, 2, 3, 1, 2, 3, 4, 8, 9, 10, 11, 9, 10, 11, 12 @@ -189,15 +167,14 @@ subpel_h_shuf4: db 0, 1, 2, 3, 1, 2, 3, 4, 8, 9, 10, 11, 9, 10, 11, 1 subpel_h_shufA: db 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6 subpel_h_shufB: db 4, 5, 6, 7, 5, 6, 7, 8, 6, 7, 8, 9, 7, 8, 9, 10 subpel_h_shufC: db 8, 9, 10, 11, 9, 10, 11, 12, 10, 11, 12, 13, 11, 12, 13, 14 -bilin_h_shuf4: db 1, 0, 2, 1, 3, 2, 4, 3, 9, 8, 10, 9, 11, 10, 12, 11 -bilin_h_shuf8: db 1, 0, 2, 1, 3, 2, 4, 3, 5, 4, 6, 5, 7, 6, 8, 7 -bilin_v_shuf4: db 4, 0, 5, 1, 6, 2, 7, 3, 8, 4, 9, 5, 10, 6, 11, 7 +bilin_h_shuf4: db 0, 1, 1, 2, 2, 3, 3, 4, 8, 9, 9, 10, 10, 11, 11, 12 +bilin_v_shuf4: db 0, 4, 1, 5, 2, 6, 3, 7, 4, 8, 5, 9, 6, 10, 7, 11 blend_shuf: db 0, 1, 0, 1, 0, 1, 0, 1, 2, 3, 2, 3, 2, 3, 2, 3 rescale_mul: dd 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 -resize_shuf: db 0, 0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 7, 7, 7, 7, 7 resize_permA: dd 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30 resize_permB: dd 1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31 resize_permC: dd 0, 4, 8, 12 +resize_shuf: db 0, 0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 7, 7, 7, 7, 7 pb_02461357: db 0, 2, 4, 6, 1, 3, 5, 7 wm_420_perm64: dq 0xfedcba9876543210 @@ -205,6 +182,8 @@ wm_sign: dd 0x40804080, 0xc0c0c0c0, 0x40404040 pb_8x0_8x8: times 8 db 0 times 8 db 8 +pb_4: times 4 db 4 +pb_32: times 4 db 32 pb_127: times 4 db 127 pw_m128 times 2 dw -128 pw_m256: times 2 dw -256 @@ -216,7 +195,6 @@ pd_32: dd 32 pd_34: dd 34 pd_63: dd 63 pd_512: dd 512 -pd_32768: dd 32768 %define pb_m64 (wm_sign+4) %define pb_64 (wm_sign+8) @@ -289,8 +267,10 @@ BASE_JMP_TABLE put, avx512icl, 2, 4, 8, 16, 32, 64, 128 BASE_JMP_TABLE prep, avx512icl, 4, 8, 16, 32, 64, 128 HV_JMP_TABLE put, bilin, avx512icl, 7, 2, 4, 8, 16, 32, 64, 128 HV_JMP_TABLE prep, bilin, avx512icl, 7, 4, 8, 16, 32, 64, 128 +HV_JMP_TABLE put, 6tap, avx512icl, 2, 2, 4, 8, 16, 32, 64, 128 HV_JMP_TABLE put, 8tap, avx512icl, 3, 2, 4, 8, 16, 32, 64, 128 -HV_JMP_TABLE prep, 8tap, avx512icl, 7, 4, 8, 16, 32, 64, 128 +HV_JMP_TABLE prep, 6tap, avx512icl, 2, 4, 8, 16, 32, 64, 128 +HV_JMP_TABLE prep, 8tap, avx512icl, 3, 4, 8, 16, 32, 64, 128 BIDIR_JMP_TABLE avg, avx512icl, 4, 8, 16, 32, 64, 128 BIDIR_JMP_TABLE w_avg, avx512icl, 4, 8, 16, 32, 64, 128 BIDIR_JMP_TABLE mask, avx512icl, 4, 8, 16, 32, 64, 128 @@ -401,9 +381,9 @@ cglobal put_bilin_8bpc, 4, 8, 0, dst, ds, src, ss, w, h, mxy .h: ; (16 * src[x] + (mx * (src[x + 1] - src[x])) + 8) >> 4 ; = ((16 - mx) * src[x] + mx * src[x + 1] + 8) >> 4 - imul mxyd, 0xff01 - vbroadcasti128 m4, [bilin_h_shuf8] - add mxyd, 16 << 8 + imul mxyd, 255 + vbroadcasti128 m4, [bilin_h_perm16] + add mxyd, 16 vpbroadcastw m5, mxyd mov mxyd, r7m ; my test mxyd, mxyd @@ -526,9 +506,9 @@ cglobal put_bilin_8bpc, 4, 8, 0, dst, ds, src, ss, w, h, mxy RET .v: movzx wd, word [r7+wq*2+table_offset(put, _bilin_v)] - imul mxyd, 0xff01 + imul mxyd, 255 vpbroadcastd m5, [pw_2048] - add mxyd, 16 << 8 + add mxyd, 16 add wq, r7 vpbroadcastw m4, mxyd jmp wq @@ -539,7 +519,7 @@ cglobal put_bilin_8bpc, 4, 8, 0, dst, ds, src, ss, w, h, mxy lea srcq, [srcq+ssq*2] pinsrw xmm0, xmm1, [srcq+ssq*0], 0 ; 2 1 pshuflw xmm1, xmm1, q2301 ; 1 0 - punpcklbw xmm1, xmm0, xmm1 + punpcklbw xmm1, xmm0 pmaddubsw xmm1, xm4 pmulhrsw xmm1, xm5 packuswb xmm1, xmm1 @@ -552,11 +532,11 @@ cglobal put_bilin_8bpc, 4, 8, 0, dst, ds, src, ss, w, h, mxy .v_w4: movd xmm0, [srcq+ssq*0] .v_w4_loop: - vpbroadcastd xmm1, [srcq+ssq*1] + vpbroadcastd xmm2, [srcq+ssq*1] lea srcq, [srcq+ssq*2] - vpblendd xmm2, xmm1, xmm0, 0x01 ; 0 1 + vpblendd xmm1, xmm2, xmm0, 0x01 ; 0 1 vpbroadcastd xmm0, [srcq+ssq*0] - vpblendd xmm1, xmm0, 0x02 ; 1 2 + vpblendd xmm2, xmm0, 0x02 ; 1 2 punpcklbw xmm1, xmm2 pmaddubsw xmm1, xm4 pmulhrsw xmm1, xm5 @@ -570,11 +550,11 @@ cglobal put_bilin_8bpc, 4, 8, 0, dst, ds, src, ss, w, h, mxy .v_w8: movq xmm0, [srcq+ssq*0] .v_w8_loop: - movq xmm3, [srcq+ssq*1] + movq xmm2, [srcq+ssq*1] lea srcq, [srcq+ssq*2] - punpcklbw xmm1, xmm3, xmm0 + punpcklbw xmm1, xmm0, xmm2 movq xmm0, [srcq+ssq*0] - punpcklbw xmm2, xmm0, xmm3 + punpcklbw xmm2, xmm0 pmaddubsw xmm1, xm4 pmaddubsw xmm2, xm4 pmulhrsw xmm1, xm5 @@ -589,11 +569,11 @@ cglobal put_bilin_8bpc, 4, 8, 0, dst, ds, src, ss, w, h, mxy .v_w16: movu xmm0, [srcq+ssq*0] .v_w16_loop: - vbroadcasti128 ymm2, [srcq+ssq*1] + vbroadcasti128 ymm3, [srcq+ssq*1] lea srcq, [srcq+ssq*2] - vpblendd ymm3, ymm2, ymm0, 0x0f ; 0 1 + vpblendd ymm2, ymm3, ymm0, 0x0f ; 0 1 vbroadcasti128 ymm0, [srcq+ssq*0] - vpblendd ymm2, ymm2, ymm0, 0xf0 ; 1 2 + vpblendd ymm3, ymm0, 0xf0 ; 1 2 punpcklbw ymm1, ymm2, ymm3 punpckhbw ymm2, ymm3 pmaddubsw ymm1, ym4 @@ -612,11 +592,11 @@ cglobal put_bilin_8bpc, 4, 8, 0, dst, ds, src, ss, w, h, mxy movu ym0, [srcq+ssq*0] kxnorb k1, k1, k1 .v_w32_loop: - vbroadcasti32x8 m2, [srcq+ssq*1] + vbroadcasti32x8 m3, [srcq+ssq*1] lea srcq, [srcq+ssq*2] - vpblendmd m3{k1}, m2, m0 ; 0 1 + vpblendmd m2{k1}, m3, m0 ; 0 1 vbroadcasti32x8 m0, [srcq+ssq*0] - vpblendmd m2{k1}, m0, m2 ; 1 2 + vpblendmd m3{k1}, m0, m3 ; 1 2 punpcklbw m1, m2, m3 punpckhbw m2, m3 pmaddubsw m1, m4 @@ -635,18 +615,18 @@ cglobal put_bilin_8bpc, 4, 8, 0, dst, ds, src, ss, w, h, mxy .v_w64_loop: movu m3, [srcq+ssq*1] lea srcq, [srcq+ssq*2] - punpcklbw m1, m3, m0 - punpckhbw m6, m3, m0 + punpcklbw m1, m0, m3 + punpckhbw m6, m0, m3 movu m0, [srcq+ssq*0] pmaddubsw m1, m4 pmaddubsw m6, m4 - punpcklbw m2, m0, m3 - punpckhbw m7, m0, m3 + punpcklbw m2, m3, m0 + punpckhbw m3, m0 pmaddubsw m2, m4 - pmaddubsw m7, m4 - REPX {pmulhrsw x, m5}, m1, m6, m2, m7 + pmaddubsw m3, m4 + REPX {pmulhrsw x, m5}, m1, m6, m2, m3 packuswb m1, m6 - packuswb m2, m7 + packuswb m2, m3 mova [dstq+dsq*0], m1 mova [dstq+dsq*1], m2 lea dstq, [dstq+dsq*2] @@ -660,13 +640,13 @@ cglobal put_bilin_8bpc, 4, 8, 0, dst, ds, src, ss, w, h, mxy add srcq, ssq movu m2, [srcq+64*0] movu m3, [srcq+64*1] - punpcklbw m6, m2, m0 + punpcklbw m6, m0, m2 pmaddubsw m6, m4 - punpckhbw m0, m2, m0 + punpckhbw m0, m2 pmaddubsw m0, m4 - punpcklbw m7, m3, m1 + punpcklbw m7, m1, m3 pmaddubsw m7, m4 - punpckhbw m1, m3, m1 + punpckhbw m1, m3 pmaddubsw m1, m4 REPX {pmulhrsw x, m5}, m6, m0, m7, m1 packuswb m6, m0 @@ -1005,8 +985,8 @@ cglobal prep_bilin_8bpc, 3, 7, 0, tmp, src, stride, w, h, mxy, stride3 .h: ; 16 * src[x] + (mx * (src[x + 1] - src[x])) ; = (16 - mx) * src[x] + mx * src[x + 1] - imul mxyd, 0xff01 - add mxyd, 16 << 8 + imul mxyd, 255 + add mxyd, 16 vpbroadcastw m5, mxyd mov mxyd, r6m ; my test mxyd, mxyd @@ -1032,7 +1012,7 @@ cglobal prep_bilin_8bpc, 3, 7, 0, tmp, src, stride, w, h, mxy, stride3 jg .h_w4_loop RET .h_w8: - vbroadcasti32x4 m4, [bilin_h_shuf8] + vbroadcasti32x4 m4, [bilin_h_perm16] .h_w8_loop: movu xmm0, [srcq+strideq*0] vinserti32x4 ym0, ymm0, [srcq+strideq*1], 1 @@ -1127,8 +1107,8 @@ cglobal prep_bilin_8bpc, 3, 7, 0, tmp, src, stride, w, h, mxy, stride3 .v: WIN64_SPILL_XMM 7 movzx wd, word [t2+wq*2+table_offset(prep, _bilin_v)] - imul mxyd, 0xff01 - add mxyd, 16 << 8 + imul mxyd, 255 + add mxyd, 16 add wq, t2 lea stride3q, [strideq*3] vpbroadcastw m6, mxyd @@ -1218,11 +1198,11 @@ cglobal prep_bilin_8bpc, 3, 7, 0, tmp, src, stride, w, h, mxy, stride3 .v_w64_loop: vpermq m1, m5, [srcq+strideq*1] lea srcq, [srcq+strideq*2] - punpcklbw m4, m1, m0 - punpckhbw m2, m1, m0 + punpcklbw m4, m0, m1 + punpckhbw m2, m0, m1 vpermq m0, m5, [srcq+strideq*0] - punpcklbw m3, m0, m1 - punpckhbw m1, m0, m1 + punpcklbw m3, m1, m0 + punpckhbw m1, m0 pmaddubsw m4, m6 pmaddubsw m2, m6 pmaddubsw m3, m6 @@ -1243,28 +1223,28 @@ cglobal prep_bilin_8bpc, 3, 7, 0, tmp, src, stride, w, h, mxy, stride3 vpermq m2, m5, [srcq+strideq*1+ 0] vpermq m3, m5, [srcq+strideq*1+64] lea srcq, [srcq+strideq*2] - punpcklbw m4, m2, m0 - punpckhbw m0, m2, m0 + punpcklbw m4, m0, m2 + punpckhbw m0, m2 pmaddubsw m4, m6 pmaddubsw m0, m6 mova [tmpq+64*0], m4 mova [tmpq+64*1], m0 - punpcklbw m4, m3, m1 - punpckhbw m1, m3, m1 + punpcklbw m4, m1, m3 + punpckhbw m1, m3 pmaddubsw m4, m6 pmaddubsw m1, m6 mova [tmpq+64*2], m4 mova [tmpq+64*3], m1 vpermq m0, m5, [srcq+strideq*0+ 0] vpermq m1, m5, [srcq+strideq*0+64] - punpcklbw m4, m0, m2 - punpckhbw m2, m0, m2 + punpcklbw m4, m2, m0 + punpckhbw m2, m0 pmaddubsw m4, m6 pmaddubsw m2, m6 mova [tmpq+64*4], m4 mova [tmpq+64*5], m2 - punpcklbw m4, m1, m3 - punpckhbw m3, m1, m3 + punpcklbw m4, m3, m1 + punpckhbw m3, m1 pmaddubsw m4, m6 pmaddubsw m3, m6 mova [tmpq+64*6], m4 @@ -1308,7 +1288,7 @@ cglobal prep_bilin_8bpc, 3, 7, 0, tmp, src, stride, w, h, mxy, stride3 jg .hv_w4_loop RET .hv_w8: - vbroadcasti32x4 m4, [bilin_h_shuf8] + vbroadcasti32x4 m4, [bilin_h_perm16] vbroadcasti32x4 m0, [srcq+strideq*0] pshufb m0, m4 pmaddubsw m0, m5 @@ -1448,7 +1428,7 @@ cglobal prep_bilin_8bpc, 3, 7, 0, tmp, src, stride, w, h, mxy, stride3 %assign FILTER_SMOOTH (1*15 << 16) | 4*15 %assign FILTER_SHARP (2*15 << 16) | 3*15 -%macro FN 4 ; fn, type, type_h, type_v +%macro FN 4-5 ; fn, type, type_h, type_v, jmp_to cglobal %1_%2_8bpc mov t0d, FILTER_%3 %ifidn %3, %4 @@ -1456,8 +1436,8 @@ cglobal %1_%2_8bpc %else mov t1d, FILTER_%4 %endif -%ifnidn %2, regular ; skip the jump in the last filter - jmp mangle(private_prefix %+ _%1_8bpc %+ SUFFIX) +%if %0 == 5 ; skip the jump in the last filter + jmp mangle(private_prefix %+ _%5 %+ SUFFIX) %endif %endmacro @@ -1489,24 +1469,22 @@ DECLARE_REG_TMP 4, 5 DECLARE_REG_TMP 7, 8 %endif +; Due to the use of vpdpbusd (which does 4 pixels per instruction) in +; the horizontal filter, 6-tap is only used for the vertical filter. %define PUT_8TAP_FN FN put_8tap, - -PUT_8TAP_FN sharp, SHARP, SHARP -PUT_8TAP_FN sharp_smooth, SHARP, SMOOTH -PUT_8TAP_FN smooth_sharp, SMOOTH, SHARP -PUT_8TAP_FN smooth, SMOOTH, SMOOTH -PUT_8TAP_FN sharp_regular, SHARP, REGULAR -PUT_8TAP_FN regular_sharp, REGULAR, SHARP -PUT_8TAP_FN smooth_regular, SMOOTH, REGULAR -PUT_8TAP_FN regular_smooth, REGULAR, SMOOTH +PUT_8TAP_FN sharp_smooth, SHARP, SMOOTH, put_6tap_8bpc +PUT_8TAP_FN sharp_regular, SHARP, REGULAR, put_6tap_8bpc +PUT_8TAP_FN smooth, SMOOTH, SMOOTH, put_6tap_8bpc +PUT_8TAP_FN smooth_regular, SMOOTH, REGULAR, put_6tap_8bpc +PUT_8TAP_FN regular_smooth, REGULAR, SMOOTH, put_6tap_8bpc PUT_8TAP_FN regular, REGULAR, REGULAR -cglobal put_8tap_8bpc, 4, 9, 0, dst, ds, src, ss, w, h, mx, my, ss3 +cglobal put_6tap_8bpc, 4, 9, 0, dst, ds, src, ss, w, h, mx, my, ns %define base r8-put_avx512icl imul mxd, mxm, 0x010101 - add mxd, t0d ; 8tap_h, mx, 4tap_h + add mxd, t0d ; 6tap_h, mx, 4tap_h imul myd, mym, 0x010101 - add myd, t1d ; 8tap_v, my, 4tap_v + add myd, t1d ; 6tap_v, my, 4tap_v lea r8, [put_avx512icl] movsxd wq, wm movifnidn hd, hm @@ -1514,6 +1492,7 @@ cglobal put_8tap_8bpc, 4, 9, 0, dst, ds, src, ss, w, h, mx, my, ss3 jnz .h test myd, 0xf00 jnz .v +.put: tzcnt wd, wd movzx wd, word [r8+wq*2+table_offset(put,)] add wq, r8 @@ -1523,158 +1502,577 @@ cglobal put_8tap_8bpc, 4, 9, 0, dst, ds, src, ss, w, h, mx, my, ss3 pop r8 %endif jmp wq -.h: - test myd, 0xf00 - jnz .hv - vpbroadcastd m5, [pd_34] ; 2 + (8 << 2) - WIN64_SPILL_XMM 11 - cmp wd, 4 - jl .h_w2 - vbroadcasti128 m6, [subpel_h_shufA] - je .h_w4 - tzcnt wd, wd - vbroadcasti128 m7, [subpel_h_shufB] - vbroadcasti128 m8, [subpel_h_shufC] - shr mxd, 16 - sub srcq, 3 - movzx wd, word [r8+wq*2+table_offset(put, _8tap_h)] - vpbroadcastd m9, [base+mxq*8+subpel_filters+0] - vpbroadcastd m10, [base+mxq*8+subpel_filters+4] - add wq, r8 - jmp wq -.h_w2: - movzx mxd, mxb - dec srcq - mova xmm4, [subpel_h_shuf4] - vpbroadcastd xmm3, [base+mxq*8+subpel_filters+2] -.h_w2_loop: - movq xmm0, [srcq+ssq*0] - movhps xmm0, [srcq+ssq*1] +.v: + movzx mxd, myb + shr myd, 16 + cmp hd, 6 + cmovs myd, mxd + tzcnt r6d, wd + movzx r6d, word [r8+r6*2+table_offset(put, _6tap_v)] + vpbroadcastd m6, [pw_512] + lea myq, [base+subpel_filters+1+myq*8] + vpbroadcastw m7, [myq+0] + add r6, r8 + vpbroadcastw m8, [myq+2] + mov nsq, ssq + vpbroadcastw m9, [myq+4] + neg nsq + jmp r6 +.v_w2: + movd xmm2, [srcq+nsq*2] + pinsrw xmm2, [srcq+nsq*1], 2 + pinsrw xmm2, [srcq+ssq*0], 4 + pinsrw xmm2, [srcq+ssq*1], 6 ; 0 1 2 3 lea srcq, [srcq+ssq*2] - pshufb xmm0, xmm4 - mova xmm1, xm5 - vpdpbusd xmm1, xmm0, xmm3 - packssdw xmm0, xmm1, xmm1 - psraw xmm0, 6 - packuswb xmm0, xm0 - pextrw [dstq+dsq*0], xmm0, 0 - pextrw [dstq+dsq*1], xmm0, 1 + vpbroadcastd xmm0, [srcq+ssq*0] + palignr xmm3, xmm0, xmm2, 4 ; 1 2 3 4 + punpcklbw xmm1, xmm2, xmm3 ; 01 12 + punpckhbw xmm2, xmm3 ; 23 34 +.v_w2_loop: + vpbroadcastd xmm4, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + pmaddubsw xmm3, xmm1, xm7 ; a0 b0 + mova xmm1, xmm2 + pmaddubsw xmm2, xm8 ; a1 b1 + paddw xmm3, xmm2 + vpblendd xmm2, xmm0, xmm4, 0x02 ; 4 5 + vpbroadcastd xmm0, [srcq+ssq*0] + vpblendd xmm4, xmm0, 0x02 ; 5 6 + punpcklbw xmm2, xmm4 ; 67 78 + pmaddubsw xmm4, xmm2, xm9 ; a3 b3 + paddw xmm3, xmm4 + pmulhrsw xmm3, xm6 + packuswb xmm3, xmm3 + pextrw [dstq+dsq*0], xmm3, 0 + pextrw [dstq+dsq*1], xmm3, 2 lea dstq, [dstq+dsq*2] sub hd, 2 - jg .h_w2_loop + jg .v_w2_loop RET -.h_w4: - movzx mxd, mxb - dec srcq - vpbroadcastd xmm3, [base+mxq*8+subpel_filters+2] -.h_w4_loop: - movq xmm0, [srcq+ssq*0] - movq xmm1, [srcq+ssq*1] +.v_w4: + movd xmm2, [srcq+nsq*2] + pinsrd xmm2, [srcq+nsq*1], 1 + pinsrd xmm2, [srcq+ssq*0], 2 + pinsrd xmm2, [srcq+ssq*1], 3 ; 0 1 2 3 lea srcq, [srcq+ssq*2] - pshufb xmm0, xm6 - pshufb xmm1, xm6 - mova xmm2, xm5 - vpdpbusd xmm2, xmm0, xmm3 - mova xmm0, xm5 - vpdpbusd xmm0, xmm1, xmm3 - packssdw xmm0, xmm2, xmm0 - psraw xmm0, 6 - packuswb xmm0, xmm0 - movd [dstq+dsq*0], xmm0 - pextrd [dstq+dsq*1], xmm0, 1 + vpbroadcastd xmm0, [srcq+ssq*0] + palignr xmm3, xmm0, xmm2, 4 ; 1 2 3 4 + punpcklbw xmm1, xmm2, xmm3 ; 01 12 + punpckhbw xmm2, xmm3 ; 23 34 +.v_w4_loop: + vpbroadcastd xmm4, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + pmaddubsw xmm3, xmm1, xm7 ; a0 b0 + mova xmm1, xmm2 + pmaddubsw xmm2, xm8 ; a1 b1 + paddw xmm3, xmm2 + vpblendd xmm2, xmm0, xmm4, 0x02 ; 4 5 + vpbroadcastd xmm0, [srcq+ssq*0] + vpblendd xmm4, xmm0, 0x02 ; 5 6 + punpcklbw xmm2, xmm4 ; 45 56 + pmaddubsw xmm4, xmm2, xm9 ; a2 b2 + paddw xmm3, xmm4 + pmulhrsw xmm3, xm6 + packuswb xmm3, xmm3 + movd [dstq+dsq*0], xmm3 + pextrd [dstq+dsq*1], xmm3, 1 lea dstq, [dstq+dsq*2] sub hd, 2 - jg .h_w4_loop + jg .v_w4_loop RET -.h_w8: - movu xm0, [srcq+ssq*0] - vinserti32x4 ym0, [srcq+ssq*1], 1 +.v_w8: + movq xmm1, [srcq+nsq*2] + vpbroadcastq ymm3, [srcq+nsq*1] + vpbroadcastq ymm2, [srcq+ssq*0] + vpbroadcastq ymm4, [srcq+ssq*1] lea srcq, [srcq+ssq*2] - WRAP_YMM PUT_8TAP_H 0, 1, 2, 3 - vpmovuswb xm0, ym0 - movq [dstq+dsq*0], xm0 - movhps [dstq+dsq*1], xm0 + vpbroadcastq ymm0, [srcq+ssq*0] + vpblendd ymm1, ymm3, 0x30 + vpblendd ymm3, ymm2, 0x30 + punpcklbw ymm1, ymm3 ; 01 12 + vpblendd ymm2, ymm4, 0x30 + vpblendd ymm4, ymm0, 0x30 + punpcklbw ymm2, ymm4 ; 23 34 +.v_w8_loop: + vpbroadcastq ymm4, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + pmaddubsw ymm3, ymm1, ym7 ; a0 b0 + mova ymm1, ymm2 + pmaddubsw ymm2, ym8 ; a1 b1 + paddw ymm3, ymm2 + vpblendd ymm2, ymm0, ymm4, 0x30 + vpbroadcastq ymm0, [srcq+ssq*0] + vpblendd ymm4, ymm0, 0x30 + punpcklbw ymm2, ymm4 ; 45 56 + pmaddubsw ymm4, ymm2, ym9 ; a2 b2 + paddw ymm3, ymm4 + pmulhrsw ymm3, ym6 + vextracti128 xmm4, ymm3, 1 + packuswb xmm3, xmm4 + movq [dstq+dsq*0], xmm3 + movhps [dstq+dsq*1], xmm3 lea dstq, [dstq+dsq*2] sub hd, 2 - jg .h_w8 + jg .v_w8_loop + vzeroupper RET -.h_w16: - mova m6, [spel_h_perm16a] - mova m7, [spel_h_perm16b] - mova m8, [spel_h_perm16c] -.h_w16_loop: - movu ym0, [srcq+ssq*0] +.v_w16: + mova m5, [spel_v_perm16a] + vbroadcasti32x4 m1, [srcq+nsq*2] + vbroadcasti32x4 ym3, [srcq+nsq*1] + mov r6d, 0x0f + vbroadcasti32x4 m2, [srcq+ssq*0] + kmovb k1, r6d + vbroadcasti32x4 ym4, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + vbroadcasti32x4 m0, [srcq+ssq*0] + vshufpd m1{k1}, m3, m2, 0xcc + vshufpd m2{k1}, m4, m0, 0xcc + vpermb m1, m5, m1 ; 01 12 + vpermb m2, m5, m2 ; 23 34 +.v_w16_loop: + vbroadcasti32x4 ym4, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + pmaddubsw m3, m1, m7 ; a0 b0 + mova m1, m2 + pmaddubsw m2, m8 ; a1 b1 + paddw m3, m2 + mova m2, m0 + vbroadcasti32x4 m0, [srcq+ssq*0] + vshufpd m2{k1}, m4, m0, 0xcc + vpermb m2, m5, m2 ; 45 56 + pmaddubsw m4, m2, m9 ; a2 b2 + paddw m3, m4 + pmulhrsw m3, m6 + vextracti32x8 ym4, m3, 1 + packuswb ym3, ym4 + mova [dstq+dsq*0], xm3 + vextracti32x4 [dstq+dsq*1], ym3, 1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .v_w16_loop + RET +.v_w32: + mova m10, [spel_v_perm32] + pmovzxbq m5, [pb_02461357] + vpshrdw m11, m10, m10, 8 + movu ym0, [srcq+nsq*2] + vinserti32x8 m0, [srcq+nsq*1], 1 + vpermb m1, m10, m0 ; 01 + vinserti32x8 m0, [srcq+ssq*0], 0 + vpermb m2, m11, m0 ; 12 vinserti32x8 m0, [srcq+ssq*1], 1 lea srcq, [srcq+ssq*2] - PUT_8TAP_H 0, 1, 2, 3, 1 - vpmovuswb ym0, m0 - mova [dstq+dsq*0], xm0 - vextracti128 [dstq+dsq*1], ym0, 1 + vpermb m3, m10, m0 ; 23 + vinserti32x8 m0, [srcq+ssq*0], 0 + vpermb m4, m11, m0 ; 34 +.v_w32_loop: + vinserti32x8 m0, [srcq+ssq*1], 1 + lea srcq, [srcq+ssq*2] + pmaddubsw m12, m1, m7 + mova m1, m3 + pmaddubsw m13, m2, m7 + mova m2, m4 + pmaddubsw m14, m3, m8 + vpermb m3, m10, m0 ; 45 + vinserti32x8 m0, [srcq+ssq*0], 0 + pmaddubsw m15, m4, m8 + vpermb m4, m11, m0 ; 56 + paddw m12, m14 + pmaddubsw m14, m3, m9 + paddw m13, m15 + pmaddubsw m15, m4, m9 + paddw m12, m14 + paddw m13, m15 + pmulhrsw m12, m6 + pmulhrsw m13, m6 + packuswb m12, m13 + vpermq m12, m5, m12 + mova [dstq+dsq*0], ym12 + vextracti32x8 [dstq+dsq*1], m12, 1 lea dstq, [dstq+dsq*2] sub hd, 2 - jg .h_w16_loop + jg .v_w32_loop RET -.h_w32: - movu ym0, [srcq+ssq*0+8*0] - vinserti32x8 m0, [srcq+ssq*1+8*0], 1 - movu ym1, [srcq+ssq*0+8*1] - vinserti32x8 m1, [srcq+ssq*1+8*1], 1 +.v_w64: +.v_w128: + lea r6d, [hq+wq*4-256] +.v_loop0: + movu m2, [srcq+nsq*2] + movu m4, [srcq+nsq*1] + lea r4, [srcq+ssq*2] + movu m11, [srcq+ssq*0] + movu m13, [srcq+ssq*1] + mov r7, dstq + movu m0, [r4 +ssq*0] + punpcklbw m1, m2, m4 ; 01l + punpckhbw m2, m4 ; 01h + punpcklbw m3, m4, m11 ; 12l + punpckhbw m4, m11 ; 12h + punpcklbw m10, m11, m13 ; 23l + punpckhbw m11, m13 ; 23h + punpcklbw m12, m13, m0 ; 34l + punpckhbw m13, m0 ; 34h +.v_loop: + movu m5, [r4+ssq*1] + pmaddubsw m14, m1, m7 ; a0l + mova m1, m10 + pmaddubsw m10, m8 ; a1l + lea r4, [r4+ssq*2] + pmaddubsw m15, m2, m7 ; a0h + mova m2, m11 + pmaddubsw m11, m8 ; a1h + paddw m14, m10 + punpcklbw m10, m0, m5 ; 45l + paddw m15, m11 + punpckhbw m11, m0, m5 ; 45h + pmaddubsw m0, m10, m9 ; a2l + paddw m14, m0 + pmaddubsw m0, m11, m9 ; a2h + paddw m15, m0 + movu m0, [r4+ssq*0] + pmulhrsw m14, m6 + pmulhrsw m15, m6 + packuswb m14, m15 + pmaddubsw m15, m3, m7 ; b0l + mova m3, m12 + pmaddubsw m12, m8 ; b1l + mova [r7+dsq*0], m14 + pmaddubsw m14, m4, m7 ; b0h + mova m4, m13 + pmaddubsw m13, m8 ; b1h + paddw m15, m12 + punpcklbw m12, m5, m0 ; 56l + paddw m14, m13 + punpckhbw m13, m5, m0 ; 56h + pmaddubsw m5, m12, m9 ; b2l + paddw m15, m5 + pmaddubsw m5, m13, m9 ; b2h + paddw m14, m5 + pmulhrsw m15, m6 + pmulhrsw m14, m6 + packuswb m15, m14 + mova [r7+dsq*1], m15 + lea r7, [r7+dsq*2] + sub hd, 2 + jg .v_loop + add srcq, 64 + add dstq, 64 + movzx hd, r6b + sub r6d, 256 + jg .v_loop0 + RET +.h: + test myd, 0xf00 + jz mangle(private_prefix %+ _put_8tap_8bpc_avx512icl).h2 +.hv: + vpbroadcastd m9, [pd_34] + mova xm10, [spel_hv_end] + pxor xm0, xm0 + cmp wd, 4 + jg .hv_w8 + movzx mxd, mxb + dec srcq + vpbroadcastd m7, [base+subpel_filters+mxq*8+2] + movzx mxd, myb + shr myd, 16 + cmp hd, 6 + cmovs myd, mxd + vpbroadcastq ym1, [base+subpel_filters+1+myq*8] + mov nsq, ssq + punpcklbw ym0, ym1 + neg nsq + psraw ym0, 2 ; << 6 + pshufd ym11, ym0, q0000 + pshufd ym12, ym0, q1111 + pshufd ym13, ym0, q2222 + cmp wd, 4 + je .hv_w4 + vbroadcasti128 ym5, [subpel_h_shuf4] + movq xmm0, [srcq+nsq*2] + movhps xmm0, [srcq+nsq*1] + movq xmm2, [srcq+ssq*0] + movhps xmm2, [srcq+ssq*1] lea srcq, [srcq+ssq*2] - PUT_8TAP_H 0, 2, 3, 4 - PUT_8TAP_H 1, 4, 3, 2 - packuswb m0, m1 - mova [dstq+dsq*0], ym0 - vextracti32x8 [dstq+dsq*1], m0, 1 + vpbroadcastq ymm1, [srcq+ssq*0] + vpblendd ymm0, ymm1, 0x30 + pshufb xmm2, xm5 ; 2 3 + pshufb ymm0, ym5 ; 0 1 4 + mova xmm1, xm9 + vpdpbusd xmm1, xmm2, xm7 + mova ymm2, ym9 + vpdpbusd ymm2, ymm0, ym7 + packssdw ymm2, ymm1 + psraw ymm2, 2 + vextracti128 xmm0, ymm2, 1 + vzeroupper + palignr xmm0, xmm2, 4 + punpcklwd xmm1, xmm2, xmm0 ; 01 12 + punpckhwd xmm2, xmm0 ; 23 34 +.hv_w2_loop: + movq xmm3, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + movhps xmm3, [srcq+ssq*0] + pmaddwd xmm4, xmm1, xm11 ; a0 b0 + mova xmm1, xmm2 + vpdpwssd xmm4, xmm2, xm12 ; a1 b1 + pshufb xmm3, xm5 + mova xmm2, xm9 + vpdpbusd xmm2, xmm3, xm7 + packssdw xmm3, xmm2, xmm2 + psraw xmm3, 2 + palignr xmm2, xmm3, xmm0, 12 + mova xmm0, xmm3 + punpcklwd xmm2, xmm3 ; 45 56 + vpdpwssd xmm4, xmm2, xm13 ; a2 b2 + packuswb xmm4, xmm4 + pshufb xmm4, xm10 + pextrw [dstq+dsq*0], xmm4, 0 + pextrw [dstq+dsq*1], xmm4, 1 lea dstq, [dstq+dsq*2] sub hd, 2 - jg .h_w32 + jg .hv_w2_loop RET -.h_w64: - movu m0, [srcq+8*0] - movu m1, [srcq+8*1] - add srcq, ssq - PUT_8TAP_H 0, 2, 3, 4 - PUT_8TAP_H 1, 4, 3, 2 - packuswb m0, m1 - mova [dstq], m0 - add dstq, dsq - dec hd - jg .h_w64 +.hv_w4: + movq xm2, [srcq+nsq*2] + vpbroadcastq ym1, [srcq+nsq*1] + vinserti32x4 ym2, [srcq+ssq*0], 1 + vinserti32x4 m1, [srcq+ssq*1], 2 ; _ 1 3 + lea srcq, [srcq+ssq*2] + vbroadcasti32x4 m5, [subpel_h_shufA] + vinserti32x4 m2, [srcq+ssq*0], 2 ; 0 2 4 + pshufb m1, m5 + mova m0, m9 + pshufb m2, m5 + mova m3, m9 + vpdpbusd m0, m1, m7 + mova ym1, [spel_hv_perm4a] + vpdpbusd m3, m2, m7 + mova ym2, [spel_hv_perm4b] + mov r6d, 0x5555 + mova ym6, [spel_hv_perm4d] + packssdw m0, m3 + kmovw k1, r6d + psraw m0, 2 ; _ 0 1 2 3 4 5 6 + vpermb ym1, ym1, ym0 ; 01 12 + vpermb m2, m2, m0 ; 23 34 +.hv_w4_loop: + movq xm3, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + vinserti32x4 ym3, [srcq+ssq*0], 1 + pmaddwd ym4, ym1, ym11 ; a0 b0 + mova ym1, ym2 + pshufb ym3, ym5 + mova ym0, ym9 + vpdpbusd ym0, ym3, ym7 + vpdpwssd ym4, ym2, ym12 ; a1 b1 + vpsraw ym2{k1}, ym0, 2 ; 5 6 + vpermb ym2, ym6, ym2 ; 45 56 + vpdpwssd ym4, ym2, ym13 ; a2 b2 + packuswb ym4, ym4 + vpermb ym4, ym10, ym4 + movd [dstq+dsq*0], xm4 + pextrd [dstq+dsq*1], xm4, 1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .hv_w4_loop RET -.h_w128: - movu m0, [srcq+8*0] - movu m2, [srcq+8*1] - movu m1, [srcq+8*8] - movu m3, [srcq+8*9] - add srcq, ssq - PUT_8TAP_H 0, 4, 11, 12 - PUT_8TAP_H 2, 12, 11, 4 - PUT_8TAP_H 1, 4, 11, 12 - PUT_8TAP_H 3, 12, 11, 4 - packuswb m0, m2 - packuswb m1, m3 - mova [dstq+64*0], m0 - mova [dstq+64*1], m1 - add dstq, dsq - dec hd - jg .h_w128 +.hv_w8: + shr mxd, 16 + sub srcq, 3 + vpbroadcastd m11, [base+subpel_filters+mxq*8+0] + vpbroadcastd m12, [base+subpel_filters+mxq*8+4] + movzx mxd, myb + shr myd, 16 + cmp hd, 6 + cmovs myd, mxd + vpbroadcastq m1, [base+subpel_filters+1+myq*8] + mov nsq, ssq + punpcklbw m0, m1 + neg nsq + psraw m0, 2 ; << 6 + pshufd m13, m0, q0000 + pshufd m14, m0, q1111 + pshufd m15, m0, q2222 + cmp wd, 8 + jne .hv_w16 + movu xm0, [srcq+nsq*2] + vinserti32x4 ym0, [srcq+nsq*1], 1 + vbroadcasti32x4 m1, [subpel_h_shufA] + vinserti32x4 m0, [srcq+ssq*0], 2 + vbroadcasti32x4 m4, [subpel_h_shufB] + vinserti32x4 m0, [srcq+ssq*1], 3 + lea srcq, [srcq+ssq*2] + vbroadcasti32x4 m7, [subpel_h_shufC] + vbroadcasti32x4 ym5, [srcq+ssq*0] + vbroadcasti32x8 m6, [subpel_h_shufA] + pshufb m1, m0, m1 ; 0 1 2 3 0123 + mova m2, m9 + vpdpbusd m2, m1, m11 + pshufb m4, m0, m4 ; 0 1 2 3 4567 + mova m1, m9 + vpdpbusd m1, m4, m11 + pshufb m0, m7 ; 0 1 2 3 89ab + pshufb ym7, ym5, ym6 ; 4 0123 4567 + mova ym3, ym9 + vpdpbusd ym3, ym7, ym11 + vbroadcasti32x8 m7, [subpel_h_shufB] + vpdpbusd m2, m4, m12 + mova m4, [spel_hv_perm8a] + pshufb ym5, ym7 ; 4 4567 89ab + vpdpbusd m1, m0, m12 + vpaddd m0, m4, [pb_32] {1to16} + vpdpbusd ym3, ym5, ym12 + mova m5, [spel_hv_perm8b] + mov r6, 0x55555555ff00 + packssdw m2, m1 + vpmovsdw xm3, ym3 + kmovq k1, r6 + psraw m2, 2 ; 0 1 2 3 + psraw xm3, 2 ; 4 + vpermb m1, m4, m2 ; 01 12 + kshiftrq k2, k1, 16 + vpermt2b m2, m0, m3 ; 23 34 +.hv_w8_loop: + vbroadcasti32x4 ym3, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + vbroadcasti32x4 m3{k1}, [srcq+ssq*0] + pmaddwd m0, m1, m13 ; a0 b0 + pshufb m1, m3, m6 ; 5 6 0123 4567 + mova m4, m9 + vpdpbusd m4, m1, m11 + pshufb m3, m7 ; 5 6 4567 89ab + vpdpwssd m0, m2, m14 ; a1 b1 + mova m1, m2 + vpdpbusd m4, m3, m12 + psraw m2{k2}, m4, 2 ; 53 64 + vpermb m2, m5, m2 ; 45 56 + vpdpwssd m0, m2, m15 ; a2 b2 + packuswb m0, m0 + vpermb m0, m10, m0 + movq [dstq+dsq*0], xm0 + movhps [dstq+dsq*1], xm0 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .hv_w8_loop + RET +.hv_w16: + movu m19, [spel_hv_perm16a] + vpbroadcastd m7, [pb_4] + lea r6d, [wq*2-32] + mova m6, [spel_hv_perm16b] + paddb m20, m7, m19 + lea r6d, [hq+r6*8] + paddb m21, m7, m20 + mova ym10, [spel_hv_end16] + paddb m7, m6 +.hv_w16_loop0: + movu ym16, [srcq+nsq*2] + vinserti32x8 m16, [srcq+nsq*1], 1 + lea r4, [srcq+ssq*2] + movu ym17, [srcq+ssq*0] + vinserti32x8 m17, [srcq+ssq*1], 1 + mov r7, dstq + movu ym18, [r4 +ssq*0] + vpermb m2, m19, m16 ; 0 1 0123 89ab + mova m1, m9 + vpermb m3, m21, m16 ; 0 1 89ab ghij + vpdpbusd m1, m2, m11 + mova m2, m9 + vpermb m4, m19, m17 ; 2 3 0123 89ab + vpdpbusd m2, m3, m12 + mova m3, m9 + vpermb m5, m21, m17 ; 2 3 89ab ghij + vpdpbusd m3, m4, m11 + mova m4, m9 + vpermb m0, m6, m18 ; 4 0145 2367 89cd abef + vpdpbusd m4, m5, m12 + mova m5, m9 + vpermb m16, m20, m16 ; 0 1 4567 cdef + vpdpbusd m5, m0, m11 + vpermb m17, m20, m17 ; 2 3 4567 cdef + vpdpbusd m1, m16, m12 + vpermb m18, m7, m18 ; 4 4589 67ab cdgh efij + vpdpbusd m2, m16, m11 + vpdpbusd m3, m17, m12 + vpdpbusd m4, m17, m11 + vpdpbusd m5, m18, m12 + packssdw m1, m2 ; 01 + packssdw m3, m4 ; 23 + REPX {psraw x, 2}, m1, m3, m5 + vpshrdd m2, m1, m3, 16 ; 12 + vpshrdd m4, m3, m5, 16 ; 34 +.hv_w16_loop: + movu ym18, [r4+ssq*1] + lea r4, [r4+ssq*2] + vinserti32x8 m18, [r4+ssq*0], 1 + pmaddwd m16, m1, m13 ; a0 + vpermb m1, m19, m18 ; 5 6 0123 89ab + pmaddwd m17, m2, m13 ; b0 + vpermb m2, m20, m18 ; 5 6 4567 cdef + mova m0, m9 + vpdpbusd m0, m1, m11 + vpermb m18, m21, m18 + mova m1, m9 + vpdpbusd m1, m2, m11 + vpdpwssd m16, m3, m14 ; a1 + vpdpwssd m17, m4, m14 ; b1 + vpdpbusd m0, m2, m12 + mova m2, m4 + vpdpbusd m1, m18, m12 + packssdw m0, m1 + mova m1, m3 + psraw m4, m0, 2 ; 5 6 + vpshrdd m3, m2, m4, 16 ; 4 5 + vpdpwssd m17, m4, m15 ; b2 + vpdpwssd m16, m3, m15 ; a2 + packuswb m16, m17 + vpermb m16, m10, m16 + mova [r7+dsq*0], xm16 + vextracti128 [r7+dsq*1], ym16, 1 + lea r7, [r7+dsq*2] + sub hd, 2 + jg .hv_w16_loop + add srcq, 16 + add dstq, 16 + movzx hd, r6b + sub r6d, 1<<8 + jg .hv_w16_loop0 + vzeroupper RET + +PUT_8TAP_FN smooth_sharp, SMOOTH, SHARP, put_8tap_8bpc +PUT_8TAP_FN regular_sharp, REGULAR, SHARP, put_8tap_8bpc +PUT_8TAP_FN sharp, SHARP, SHARP + +cglobal put_8tap_8bpc, 4, 9, 0, dst, ds, src, ss, w, h, mx, my, ss3 + imul mxd, mxm, 0x010101 + add mxd, t0d ; 8tap_h, mx, 4tap_h + imul myd, mym, 0x010101 + add myd, t1d ; 8tap_v, my, 4tap_v + lea r8, [put_avx512icl] + movsxd wq, wm + movifnidn hd, hm + test mxd, 0xf00 + jnz .h + test myd, 0xf00 + jz mangle(private_prefix %+ _put_6tap_8bpc_avx512icl).put .v: movzx mxd, myb shr myd, 16 cmp hd, 6 cmovs myd, mxd tzcnt r6d, wd + lea myq, [base+subpel_filters+myq*8] movzx r6d, word [r8+r6*2+table_offset(put, _8tap_v)] vpbroadcastd m7, [pw_512] - lea myq, [base+subpel_filters+myq*8] vpbroadcastw m8, [myq+0] - vpbroadcastw m9, [myq+2] - vpbroadcastw m10, [myq+4] - vpbroadcastw m11, [myq+6] add r6, r8 + vpbroadcastw m9, [myq+2] lea ss3q, [ssq*3] + vpbroadcastw m10, [myq+4] sub srcq, ss3q + vpbroadcastw m11, [myq+6] jmp r6 .v_w2: movd xmm2, [srcq+ssq*0] @@ -1802,7 +2200,7 @@ cglobal put_8tap_8bpc, 4, 9, 0, dst, ds, src, ss, w, h, mx, my, ss3 vzeroupper RET .v_w16: - mova m12, [spel_v_perm16] + mova m12, [spel_v_perm16a] vbroadcasti32x4 m1, [srcq+ssq*0] vbroadcasti32x4 ym4, [srcq+ssq*1] mov r6d, 0x0f @@ -1990,7 +2388,146 @@ cglobal put_8tap_8bpc, 4, 9, 0, dst, ds, src, ss, w, h, mx, my, ss3 jg .v_loop0 vzeroupper RET +.h: + test myd, 0xf00 + jnz .hv +.h2: + vpbroadcastd m5, [pd_34] ; 2 + (8 << 2) + cmp wd, 4 + jl .h_w2 + vbroadcasti128 m6, [subpel_h_shufA] + je .h_w4 + tzcnt wd, wd + vbroadcasti128 m7, [subpel_h_shufB] + vbroadcasti128 m8, [subpel_h_shufC] + shr mxd, 16 + sub srcq, 3 + movzx wd, word [r8+wq*2+table_offset(put, _8tap_h)] + vpbroadcastd m9, [base+mxq*8+subpel_filters+0] + vpbroadcastd m10, [base+mxq*8+subpel_filters+4] + add wq, r8 + jmp wq +.h_w2: + movzx mxd, mxb + dec srcq + mova xmm4, [subpel_h_shuf4] + vpbroadcastd xmm3, [base+mxq*8+subpel_filters+2] +.h_w2_loop: + movq xmm0, [srcq+ssq*0] + movhps xmm0, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + pshufb xmm0, xmm4 + mova xmm1, xm5 + vpdpbusd xmm1, xmm0, xmm3 + packssdw xmm0, xmm1, xmm1 + psraw xmm0, 6 + packuswb xmm0, xm0 + pextrw [dstq+dsq*0], xmm0, 0 + pextrw [dstq+dsq*1], xmm0, 1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .h_w2_loop + RET +.h_w4: + movzx mxd, mxb + dec srcq + vpbroadcastd xmm3, [base+mxq*8+subpel_filters+2] +.h_w4_loop: + movq xmm0, [srcq+ssq*0] + movq xmm1, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + pshufb xmm0, xm6 + pshufb xmm1, xm6 + mova xmm2, xm5 + vpdpbusd xmm2, xmm0, xmm3 + mova xmm0, xm5 + vpdpbusd xmm0, xmm1, xmm3 + packssdw xmm0, xmm2, xmm0 + psraw xmm0, 6 + packuswb xmm0, xmm0 + movd [dstq+dsq*0], xmm0 + pextrd [dstq+dsq*1], xmm0, 1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .h_w4_loop + RET +.h_w8: + movu xm0, [srcq+ssq*0] + vinserti32x4 ym0, [srcq+ssq*1], 1 + lea srcq, [srcq+ssq*2] + WRAP_YMM PUT_8TAP_H 0, 1, 2, 3 + vpmovuswb xm0, ym0 + movq [dstq+dsq*0], xm0 + movhps [dstq+dsq*1], xm0 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .h_w8 + RET +.h_w16: + mova m6, [spel_h_perm16] + vpbroadcastd m8, [pb_4] + paddb m7, m8, m6 + paddb m8, m7 +.h_w16_loop: + movu ym0, [srcq+ssq*0] + vinserti32x8 m0, [srcq+ssq*1], 1 + lea srcq, [srcq+ssq*2] + PUT_8TAP_H 0, 1, 2, 3, 1 + vpmovuswb ym0, m0 + mova [dstq+dsq*0], xm0 + vextracti128 [dstq+dsq*1], ym0, 1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .h_w16_loop + RET +.h_w32: + movu ym0, [srcq+ssq*0+8*0] + vinserti32x8 m0, [srcq+ssq*1+8*0], 1 + movu ym1, [srcq+ssq*0+8*1] + vinserti32x8 m1, [srcq+ssq*1+8*1], 1 + lea srcq, [srcq+ssq*2] + PUT_8TAP_H 0, 2, 3, 4 + PUT_8TAP_H 1, 4, 3, 2 + packuswb m0, m1 + mova [dstq+dsq*0], ym0 + vextracti32x8 [dstq+dsq*1], m0, 1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .h_w32 + RET +.h_w64: + movu m0, [srcq+8*0] + movu m1, [srcq+8*1] + add srcq, ssq + PUT_8TAP_H 0, 2, 3, 4 + PUT_8TAP_H 1, 4, 3, 2 + packuswb m0, m1 + mova [dstq], m0 + add dstq, dsq + dec hd + jg .h_w64 + RET +.h_w128: + movu m0, [srcq+8*0] + movu m2, [srcq+8*1] + movu m1, [srcq+8*8] + movu m3, [srcq+8*9] + add srcq, ssq + PUT_8TAP_H 0, 4, 11, 12 + PUT_8TAP_H 2, 12, 11, 4 + PUT_8TAP_H 1, 4, 11, 12 + PUT_8TAP_H 3, 12, 11, 4 + packuswb m0, m2 + packuswb m1, m3 + mova [dstq+64*0], m0 + mova [dstq+64*1], m1 + add dstq, dsq + dec hd + jg .h_w128 + RET .hv: + vpbroadcastd m9, [pd_34] + pxor xm0, xm0 cmp wd, 4 jg .hv_w8 movzx mxd, mxb @@ -2000,12 +2537,10 @@ cglobal put_8tap_8bpc, 4, 9, 0, dst, ds, src, ss, w, h, mx, my, ss3 shr myd, 16 cmp hd, 6 cmovs myd, mxd - vpbroadcastd m8, [pd_2] - vpbroadcastq ym0, [base+subpel_filters+myq*8] + vpbroadcastq ym1, [base+subpel_filters+myq*8] lea ss3q, [ssq*3] - vpbroadcastd ym9, [pd_32768] mov r6, srcq - punpcklbw ym0, ym8, ym0 + punpcklbw ym0, ym1 sub r6, ss3q psraw ym0, 2 ; << 6 mova xm14, [spel_hv_end] @@ -2029,9 +2564,9 @@ cglobal put_8tap_8bpc, 4, 9, 0, dst, ds, src, ss, w, h, mx, my, ss3 vpblendd ymm2, ymm4, 0xc0 ; 0 1 4 5 pshufb ymm2, ym6 pshufb ymm0, ym6 - mova ymm1, ym8 + mova ymm1, ym9 vpdpbusd ymm1, ymm2, ym7 - mova ymm2, ym8 + mova ymm2, ym9 vpdpbusd ymm2, ymm0, ym7 packssdw ymm2, ymm1, ymm2 psraw ymm2, 2 @@ -2045,14 +2580,13 @@ cglobal put_8tap_8bpc, 4, 9, 0, dst, ds, src, ss, w, h, mx, my, ss3 movq xmm4, [srcq+ssq*1] lea srcq, [srcq+ssq*2] movhps xmm4, [srcq+ssq*0] - mova xmm5, xm9 - vpdpwssd xmm5, xmm1, xm10 ; a0 b0 + pmaddwd xmm5, xmm1, xm10 ; a0 b0 mova xmm1, xmm2 vpdpwssd xmm5, xmm2, xm11 ; a1 b1 pshufb xmm4, xm6 mova xmm2, xmm3 vpdpwssd xmm5, xmm3, xm12 ; a2 b2 - mova xmm3, xm8 + mova xmm3, xm9 vpdpbusd xmm3, xmm4, xm7 packssdw xmm4, xmm3, xmm3 psraw xmm4, 2 @@ -2081,9 +2615,9 @@ cglobal put_8tap_8bpc, 4, 9, 0, dst, ds, src, ss, w, h, mx, my, ss3 vinserti32x4 m1, [srcq+ssq*0], 3 ; 0 2 4 6 pshufb m2, m6 pshufb m1, m6 - mova m0, m8 + mova m0, m9 vpdpbusd m0, m2, m7 - mova m4, m8 + mova m4, m9 vpdpbusd m4, m1, m7 mova ym1, [spel_hv_perm4a] mova ym2, [spel_hv_perm4b] @@ -2100,11 +2634,10 @@ cglobal put_8tap_8bpc, 4, 9, 0, dst, ds, src, ss, w, h, mx, my, ss3 movq xmm4, [srcq+ssq*1] lea srcq, [srcq+ssq*2] vinserti32x4 ym4, ymm4, [srcq+ssq*0], 1 - mova ym5, ym9 - vpdpwssd ym5, ym1, ym10 ; a0 b0 + pmaddwd ym5, ym1, ym10 ; a0 b0 mova ym1, ym2 pshufb ym4, ym6 - mova ym0, ym8 + mova ym0, ym9 vpdpbusd ym0, ym4, ym7 vpdpwssd ym5, ym2, ym11 ; a1 b1 mova ym2, ym3 @@ -2129,10 +2662,8 @@ cglobal put_8tap_8bpc, 4, 9, 0, dst, ds, src, ss, w, h, mx, my, ss3 shr myd, 16 cmp hd, 6 cmovs myd, mxd - vpbroadcastd m8, [pd_2] - vpbroadcastq m0, [base+subpel_filters+myq*8] - vpbroadcastd m9, [pd_32768] - punpcklbw m0, m8, m0 + vpbroadcastq m1, [base+subpel_filters+myq*8] + punpcklbw m0, m1 lea ss3q, [ssq*3] psraw m0, 2 ; << 6 pshufd m12, m0, q0000 @@ -2153,31 +2684,31 @@ cglobal put_8tap_8bpc, 4, 9, 0, dst, ds, src, ss, w, h, mx, my, ss3 vbroadcasti32x4 m4, [subpel_h_shufA] vinserti32x4 m0, zmm2, [srcq+ssq*0], 2 ; 4 5 6 _ vbroadcasti32x4 m7, [subpel_h_shufB] - vbroadcasti32x4 m17, [subpel_h_shufC] + vbroadcasti32x4 m8, [subpel_h_shufC] pshufb m1, m6, m4 ; 0 1 2 3 0123 - mova m2, m8 + mova m2, m9 vpdpbusd m2, m1, m10 pshufb m5, m6, m7 ; 0 1 2 3 4567 - mova m1, m8 + mova m1, m9 vpdpbusd m1, m5, m10 pshufb m4, m0, m4 ; 4 5 6 _ 0123 - mova m3, m8 + mova m3, m9 vpdpbusd m3, m4, m10 pshufb m7, m0, m7 ; 4 5 6 _ 4567 - mova m4, m8 + mova m4, m9 vpdpbusd m4, m7, m10 - pshufb m6, m17 + pshufb m6, m8 vpdpbusd m2, m5, m11 vpdpbusd m1, m6, m11 - pshufb m6, m0, m17 + pshufb m6, m0, m8 vpdpbusd m3, m7, m11 vpdpbusd m4, m6, m11 mova m5, [spel_hv_perm8a] - mova m0, [spel_hv_perm8b] + vpaddd m0, m5, [pb_32] {1to16} mov r6, 0x55555555ff00 packssdw m2, m1 packssdw m3, m4 - mova m18, [spel_hv_perm8c] + mova m8, [spel_hv_perm8b] psraw m2, 2 ; 0 1 2 3 psraw m3, 2 ; 4 5 6 _ vpermb m1, m5, m2 ; 01 12 @@ -2192,10 +2723,9 @@ cglobal put_8tap_8bpc, 4, 9, 0, dst, ds, src, ss, w, h, mx, my, ss3 vbroadcasti32x4 ym4, [srcq+ssq*1] lea srcq, [srcq+ssq*2] vbroadcasti32x4 m4{k1}, [srcq+ssq*0] - mova m0, m9 - vpdpwssd m0, m1, m12 ; a0 b0 + pmaddwd m0, m1, m12 ; a0 b0 pshufb m1, m4, m6 ; 7 8 0123 4567 - mova m5, m8 + mova m5, m9 vpdpbusd m5, m1, m10 pshufb m4, m7 ; 7 8 4567 89ab vpdpwssd m0, m2, m13 ; a1 b1 @@ -2204,7 +2734,7 @@ cglobal put_8tap_8bpc, 4, 9, 0, dst, ds, src, ss, w, h, mx, my, ss3 mova m2, m3 vpdpwssd m0, m3, m14 ; a2 b2 psraw m3{k2}, m5, 2 ; 75 86 - vpermb m3, m18, m3 ; 67 78 + vpermb m3, m8, m3 ; 67 78 vpdpwssd m0, m3, m15 ; a3 b3 packuswb m0, m0 vpermb zmm1, m16, m0 @@ -2216,111 +2746,652 @@ cglobal put_8tap_8bpc, 4, 9, 0, dst, ds, src, ss, w, h, mx, my, ss3 vzeroupper RET .hv_w16: - movu m7, [spel_hv_perm16a] + WIN64_SPILL_XMM 23 + movu m22, [spel_hv_perm16a] sub srcq, ss3q - mova m20, [spel_hv_perm16b] + vpbroadcastd m8, [pb_4] lea r6d, [wq*2-32] - mova m21, [spel_hv_perm16c] - mov r4, srcq - mov r7, dstq + mova m7, [spel_hv_perm16b] + paddb m20, m8, m22 mova ym16, [spel_hv_end16] + paddb m21, m8, m20 lea r6d, [hq+r6*8] + paddb m8, m7 .hv_w16_loop0: movu ym17, [srcq+ssq*0] vinserti32x8 m17, [srcq+ssq*1], 1 ; 0 1 + lea r4, [srcq+ss3q] movu ym18, [srcq+ssq*2] - add srcq, ss3q - vinserti32x8 m18, [srcq+ssq*0], 1 ; 2 3 - movu ym19, [srcq+ssq*1] - vinserti32x8 m19, [srcq+ssq*2], 1 ; 4 5 - add srcq, ss3q - vpermb m2, m7, m17 ; 0 1 0123 89ab - vpermb m0, m20, m17 ; 0 1 4567 cdef - vpermb m4, m7, m18 ; 2 3 0123 89ab - mova m1, m8 + vinserti32x8 m18, [r4 +ssq*0], 1 ; 2 3 + mov r7, dstq + movu ym19, [r4 +ssq*1] + vinserti32x8 m19, [r4 +ssq*2], 1 ; 4 5 + add r4, ss3q + vpermb m2, m22, m17 ; 0 1 0123 89ab + mova m1, m9 + vpermb m3, m21, m17 ; 0 1 89ab ghij vpdpbusd m1, m2, m10 - vpermb m5, m20, m18 ; 2 3 4567 cdef - mova m2, m8 - vpdpbusd m2, m0, m10 - vpermb m17, m21, m17 ; 0 1 89ab ghij - mova m3, m8 + mova m2, m9 + vpermb m4, m22, m18 ; 2 3 0123 89ab + vpdpbusd m2, m3, m11 + mova m3, m9 + vpermb m5, m21, m18 ; 2 3 89ab ghij vpdpbusd m3, m4, m10 - vpermb m6, m7, m19 ; 4 5 0123 89ab - mova m4, m8 - vpdpbusd m4, m5, m10 - vpermb m18, m21, m18 ; 2 3 89ab ghij - vpdpbusd m1, m0, m11 - movu ym0, [srcq+ssq*0] ; 6 - vpdpbusd m2, m17, m11 - vpermb m17, m20, m19 ; 4 5 4567 cdef - vpdpbusd m3, m5, m11 - mova m5, m8 + mova m4, m9 + vpermb m6, m22, m19 ; 4 5 0123 89ab + vpdpbusd m4, m5, m11 + mova m5, m9 + vpermb m17, m20, m17 ; 0 1 4567 cdef vpdpbusd m5, m6, m10 - mova m6, m8 - vpdpbusd m6, m17, m10 - vpdpbusd m4, m18, m11 - mova m18, [spel_hv_perm16d] - vpermb m18, m18, m0 ; 6 0145 2367 89cd abef - vpdpbusd m5, m17, m11 - vpermb m19, m21, m19 ; 4 5 89ab ghij - mova m17, m8 - vpdpbusd m17, m18, m10 - mova m18, [spel_hv_perm16e] - vpermb m0, m18, m0 ; 6 4589 67ab cdgh efij - packssdw m1, m2 ; 01 - vpdpbusd m6, m19, m11 - packssdw m3, m4 ; 23 - vpdpbusd m17, m0, m11 - psraw m1, 2 - packssdw m5, m6 ; 45 - psraw m3, 2 + mova m6, m9 + vpermb m0, m21, m19 ; 4 5 89ab ghij + vpdpbusd m1, m17, m11 + vpdpbusd m2, m17, m10 + movu ym17, [r4+ssq*0] ; 6 + vpermb m18, m20, m18 ; 2 3 4567 cdef + vpdpbusd m6, m0, m11 + vpermb m0, m7, m17 ; 6 0145 2367 89cd abef + vpdpbusd m3, m18, m11 + vpermb m19, m20, m19 ; 4 5 4567 cdef + vpdpbusd m4, m18, m10 + mova m18, m9 + vpermb m17, m8, m17 ; 6 4589 67ab cdgh efij + vpdpbusd m18, m0, m10 + packssdw m1, m2 + vpdpbusd m5, m19, m11 + vpdpbusd m6, m19, m10 + packssdw m3, m4 + vpdpbusd m18, m17, m11 + psraw m1, 2 ; 01 + psraw m3, 2 ; 23 + packssdw m5, m6 vpshrdd m2, m1, m3, 16 ; 12 - psraw m5, 2 + psraw m5, 2 ; 45 vpshrdd m4, m3, m5, 16 ; 34 - psraw m17, 2 - vpshrdd m6, m5, m17, 16 ; 56 + psraw m18, 2 + vpshrdd m6, m5, m18, 16 ; 56 .hv_w16_loop: - movu ym18, [srcq+ssq*1] - lea srcq, [srcq+ssq*2] - vinserti32x8 m18, [srcq+ssq*0], 1 + movu ym19, [r4+ssq*1] + lea r4, [r4+ssq*2] + vinserti32x8 m19, [r4+ssq*0], 1 + pmaddwd m17, m1, m12 ; a0 + vpermb m1, m22, m19 ; 7 8 0123 89ab + pmaddwd m18, m2, m12 ; b0 mova m0, m9 - vpdpwssd m0, m1, m12 ; a0 - vpermb m1, m7, m18 ; 7 8 0123 89ab - mova m17, m9 - vpdpwssd m17, m2, m12 ; b0 - vpermb m2, m20, m18 ; 7 8 4567 cdef - mova m19, m8 - vpdpbusd m19, m1, m10 - vpermb m18, m21, m18 - mova m1, m8 - vpdpbusd m1, m2, m10 - vpdpwssd m0, m3, m13 ; a1 - vpdpwssd m17, m4, m13 ; b1 - vpdpbusd m19, m2, m11 + vpermb m2, m21, m19 ; 7 8 89ab ghij + vpdpbusd m0, m1, m10 + mova m1, m9 + vpermb m19, m20, m19 ; 7 8 4567 cdef + vpdpbusd m1, m2, m11 mova m2, m4 - vpdpbusd m1, m18, m11 + vpdpwssd m17, m3, m13 ; a1 + vpdpwssd m18, m4, m13 ; b1 mova m4, m6 - vpdpwssd m0, m5, m14 ; a2 - vpdpwssd m17, m6, m14 ; b2 - packssdw m19, m1 + vpdpbusd m0, m19, m11 + vpdpbusd m1, m19, m10 + vpdpwssd m17, m5, m14 ; a2 + vpdpwssd m18, m6, m14 ; b2 + packssdw m0, m1 mova m1, m3 + psraw m6, m0, 2 ; 78 mova m3, m5 - psraw m6, m19, 2 ; 7 8 - vpshrdd m5, m4, m6, 16 ; 6 7 - vpdpwssd m17, m6, m15 ; b3 - vpdpwssd m0, m5, m15 ; a3 - packuswb m0, m17 - vpermb zmm1, m16, m0 - mova [dstq+dsq*0], xmm1 - vextracti128 [dstq+dsq*1], ymm1, 1 - lea dstq, [dstq+dsq*2] + vpshrdd m5, m4, m6, 16 ; 67 + vpdpwssd m18, m6, m15 ; b3 + vpdpwssd m17, m5, m15 ; a3 + packuswb m17, m18 + vpermb m17, m16, m17 + mova [r7+dsq*0], xm17 + vextracti128 [r7+dsq*1], ym17, 1 + lea r7, [r7+dsq*2] sub hd, 2 jg .hv_w16_loop - add r4, 16 - add r7, 16 + add srcq, 16 + add dstq, 16 + movzx hd, r6b + sub r6d, 1<<8 + jg .hv_w16_loop0 + RET + +%if WIN64 +DECLARE_REG_TMP 6, 4 +%else +DECLARE_REG_TMP 6, 7 +%endif + +%define PREP_8TAP_FN FN prep_8tap, +PREP_8TAP_FN sharp_smooth, SHARP, SMOOTH, prep_6tap_8bpc +PREP_8TAP_FN sharp_regular, SHARP, REGULAR, prep_6tap_8bpc +PREP_8TAP_FN smooth, SMOOTH, SMOOTH, prep_6tap_8bpc +PREP_8TAP_FN smooth_regular, SMOOTH, REGULAR, prep_6tap_8bpc +PREP_8TAP_FN regular_smooth, REGULAR, SMOOTH, prep_6tap_8bpc +PREP_8TAP_FN regular, REGULAR, REGULAR + +cglobal prep_6tap_8bpc, 4, 8, 0, tmp, src, ss, w, h, mx, my, ss3 +%define base r7-prep_avx512icl + imul mxd, mxm, 0x010101 + add mxd, t0d ; 6tap_h, mx, 4tap_h + imul myd, mym, 0x010101 + add myd, t1d ; 6tap_v, my, 4tap_v + lea r7, [prep_avx512icl] + movifnidn hd, hm + test mxd, 0xf00 + jnz .h + test myd, 0xf00 + jnz .v +.prep: + tzcnt wd, wd + movzx wd, word [r7+wq*2+table_offset(prep,)] + add wq, r7 + lea r6, [ssq*3] +%if WIN64 + pop r7 +%endif + jmp wq +.v: + movzx mxd, myb + shr myd, 16 + cmp hd, 4 + cmove myd, mxd + tzcnt r5d, wd + lea myq, [base+subpel_filters+1+myq*8] + movzx r5d, word [r7+r5*2+table_offset(prep, _6tap_v)] + vpbroadcastd m7, [pw_8192] + sub srcq, ssq + vpbroadcastw m8, [myq+0] + add r5, r7 + vpbroadcastw m9, [myq+2] + lea ss3q, [ssq*3] + vpbroadcastw m10, [myq+4] + sub srcq, ssq + jmp r5 +.v_w4: + movd xmm2, [srcq+ssq*0] + pinsrd xmm2, [srcq+ssq*1], 1 + vpbroadcastd ymm1, [srcq+ssq*2] + add srcq, ss3q + vpbroadcastd ymm3, [srcq+ssq*0] + vpbroadcastd ymm0, [srcq+ssq*1] + vbroadcasti128 ymm5, [deint_shuf4] + vpblendd ymm1, ymm2, 0xeb + punpcklqdq ymm3, ymm0 + vpblendd ymm1, ymm3, 0x60 ; 0 1 2 _ 2 3 4 _ + pshufb ymm1, ymm5 ; 01 12 23 34 +.v_w4_loop: + pinsrd xmm0, [srcq+ssq*2], 1 + vpbroadcastd ymm2, [srcq+ss3q ] + lea srcq, [srcq+ssq*4] + vpbroadcastd ymm3, [srcq+ssq*0] + vpblendd ymm2, ymm0, 0xeb + vpbroadcastd ymm0, [srcq+ssq*1] + punpcklqdq ymm3, ymm0 + vpblendd ymm2, ymm3, 0x60 ; 4 5 6 _ 6 7 8 _ + pshufb ymm2, ymm5 ; 45 56 67 78 + pmaddubsw ymm3, ymm1, ym8 ; a0 b0 c0 d0 + vperm2i128 ymm1, ymm2, 0x21 ; 23 34 45 56 + pmaddubsw ymm4, ymm2, ym10 ; a2 b2 c2 d2 + pmaddubsw ymm1, ym9 ; a1 b1 c1 d1 + paddw ymm3, ymm4 + paddw ymm3, ymm1 + pmulhrsw ymm3, ym7 + mova ymm1, ymm2 + mova [tmpq], ymm3 + add tmpq, 32 + sub hd, 4 + jg .v_w4_loop + vzeroupper + RET +.v_w8: + mova m6, [spel_v_perm8] + movq xm1, [srcq+ssq*0] + mov r6d, 0x3e + movq xm2, [srcq+ssq*1] + kmovb k1, r6d + vpbroadcastq ym3, [srcq+ssq*2] + add srcq, ss3q + vpunpcklqdq ym2, [srcq+ssq*0] {1to4} + vpunpcklqdq m1{k1}, m3, [srcq+ssq*1] {1to8} + movq xm0, [srcq+ssq*1] + kshiftlb k2, k1, 2 + shufpd m1, m2, 0x18 ; 0 1 2 3 4 + vpermb m1, m6, m1 ; 01 12 23 34 +.v_w8_loop: + vpbroadcastq ym3, [srcq+ss3q ] + vpunpcklqdq ym0{k1}, ym3, [srcq+ssq*2] {1to4} + lea srcq, [srcq+ssq*4] + vpbroadcastq m3, [srcq+ssq*1] + vpunpcklqdq m0{k2}, m3, [srcq+ssq*0] {1to8} + pmaddubsw m4, m1, m8 ; a0 b0 c0 d0 + vpermb m2, m6, m0 ; 45 56 67 78 + mova xm0, xm3 + vshufi32x4 m1, m2, q1032 ; 23 34 45 56 + pmaddubsw m3, m2, m10 ; a3 b3 c3 d3 + pmaddubsw m5, m1, m9 ; a2 b2 c2 d2 + mova m1, m2 + paddw m4, m3 + paddw m4, m5 + pmulhrsw m4, m7 + mova [tmpq], m4 + add tmpq, 64 + sub hd, 4 + jg .v_w8_loop + RET +.v_w16: + mova m11, [spel_v_perm16b] + vbroadcasti32x4 m1, [srcq+ssq*0] + mov r6d, 0x0f + vbroadcasti32x4 ym3, [srcq+ssq*1] + vbroadcasti32x4 m2, [srcq+ssq*2] + kmovb k1, r6d + add srcq, ss3q + vbroadcasti32x4 ym4, [srcq+ssq*0] + vbroadcasti32x4 m0, [srcq+ssq*1] + vshufpd m1{k1}, m3, m2, 0xcc + vshufpd m2{k1}, m4, m0, 0xcc + vpermb m1, m11, m1 ; 01 12 + vpermb m2, m11, m2 ; 23 34 +.v_w16_loop: + pmaddubsw m3, m1, m8 ; a0 b0 + pmaddubsw m5, m2, m9 ; a1 b1 + vbroadcasti32x4 ym6, [srcq+ssq*2] + pmaddubsw m4, m2, m8 ; c0 d0 + vbroadcasti32x4 m2, [srcq+ss3q ] + lea srcq, [srcq+ssq*4] + vshufpd m0{k1}, m6, m2, 0xcc + vbroadcasti32x4 ym6, [srcq+ssq*0] + vpermb m1, m11, m0 ; 45 56 + vbroadcasti32x4 m0, [srcq+ssq*1] + vshufpd m2{k1}, m6, m0, 0xcc + pmaddubsw m6, m1, m9 ; c1 d1 + vpermb m2, m11, m2 ; 67 78 + paddw m3, m5 + pmaddubsw m5, m1, m10 ; a2 b2 + paddw m4, m6 + pmaddubsw m6, m2, m10 ; c2 d2 + paddw m3, m5 + paddw m4, m6 + pmulhrsw m3, m7 + pmulhrsw m4, m7 + mova [tmpq+ 0], m3 + mova [tmpq+64], m4 + add tmpq, 64*2 + sub hd, 4 + jg .v_w16_loop + RET +.v_w32: + movshdup m6, [bilin_v_perm64] + movu ym16, [srcq+ssq*0] + movu ym17, [srcq+ssq*1] + movu ym18, [srcq+ssq*2] + add srcq, ss3q + movu ym19, [srcq+ssq*0] + add srcq, ssq + movu ym20, [srcq+ssq*0] + vpermt2q m16, m6, m18 ; 0 2 + vpermt2q m17, m6, m19 ; 1 3 + vpermt2q m18, m6, m20 ; 2 4 + punpcklbw m0, m16, m17 ; 01 + punpcklbw m1, m17, m18 ; 12 + punpckhbw m2, m16, m17 ; 23 + punpckhbw m3, m17, m18 ; 34 +.v_w32_loop: + movu ym16, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + movu ym17, [srcq+ssq*0] + pmaddubsw m4, m0, m8 ; a0 + mova m0, m2 + pmaddubsw m2, m9 ; a1 + vpermt2q m16, m6, m17 ; 5 6 + pmaddubsw m5, m1, m8 ; b0 + mova m1, m3 + pmaddubsw m3, m9 ; b1 + shufpd m18, m16, 0x55 ; 4 5 + paddw m4, m2 + punpcklbw m2, m18, m16 ; 45 + paddw m5, m3 + punpckhbw m3, m18, m16 ; 56 + mova m18, m16 + pmaddubsw m16, m2, m10 ; a2 + pmaddubsw m17, m3, m10 ; b2 + paddw m4, m16 + paddw m5, m17 + pmulhrsw m4, m7 + pmulhrsw m5, m7 + mova [tmpq+ 0], m4 + mova [tmpq+64], m5 + add tmpq, 64*2 + sub hd, 2 + jg .v_w32_loop + vzeroupper + RET +.v_w64: +.v_w128: + mova m6, [bilin_v_perm64] + add wd, wd + lea r6d, [hq+wq] +.v_loop0: + vpermq m12, m6, [srcq+ssq*0] + vpermq m13, m6, [srcq+ssq*1] + lea r5, [srcq+ssq*2] + vpermq m14, m6, [r5 +ssq*0] + vpermq m15, m6, [r5 +ssq*1] + lea r5, [r5+ssq*2] + vpermq m16, m6, [r5 +ssq*0] + mov r7, tmpq + punpcklbw m0, m12, m13 ; 01 + punpckhbw m12, m13 + punpcklbw m1, m13, m14 ; 12 + punpckhbw m13, m14 + punpcklbw m2, m14, m15 ; 23 + punpckhbw m14, m15 + punpcklbw m3, m15, m16 ; 34 + punpckhbw m15, m16 +.v_loop: + pmaddubsw m17, m0, m8 ; a0 + vpermq m5, m6, [r5+ssq*1] + pmaddubsw m18, m12, m8 + mova m0, m2 + pmaddubsw m2, m9 ; a1 + mova m12, m14 + pmaddubsw m14, m9 + lea r5, [r5+ssq*2] + pmaddubsw m19, m1, m8 ; b0 + pmaddubsw m20, m13, m8 + mova m1, m3 + pmaddubsw m3, m9 ; b1 + mova m13, m15 + pmaddubsw m15, m9 + paddw m17, m2 + punpcklbw m2, m16, m5 ; 67 + paddw m18, m14 + punpckhbw m14, m16, m5 + vpermq m16, m6, [r5+ssq*0] + paddw m19, m3 + pmaddubsw m3, m2, m10 ; a3 + paddw m20, m15 + pmaddubsw m15, m14, m10 + paddw m17, m3 + punpcklbw m3, m5, m16 ; 78 + pmaddubsw m4, m3, m10 ; b3 + paddw m18, m15 + punpckhbw m15, m5, m16 + pmaddubsw m5, m15, m10 + paddw m19, m4 + paddw m20, m5 + REPX {pmulhrsw x, m7}, m17, m18, m19, m20 + mova [r7+wq*0+ 0], m17 + mova [r7+wq*0+64], m18 + mova [r7+wq*1+ 0], m19 + mova [r7+wq*1+64], m20 + lea r7, [r7+wq*2] + sub hd, 2 + jg .v_loop + add srcq, 64 + add tmpq, 128 + movzx hd, r6b + sub r6d, 1<<8 + jg .v_loop0 + vzeroupper + RET +.h: + test myd, 0xf00 + jz mangle(private_prefix %+ _prep_8tap_8bpc_avx512icl).h2 +.hv: + vpbroadcastd m8, [pd_2] + vpbroadcastd m9, [pd_32] + cmp wd, 4 + jg .hv_w8 + movzx mxd, mxb + vpbroadcastd m11, [base+subpel_filters+mxq*8+2] + movzx mxd, myb + shr myd, 16 + cmp hd, 4 + cmove myd, mxd + vpbroadcastq m3, [base+subpel_filters+1+myq*8] + vbroadcasti128 m10, [subpel_h_shufA] + lea r6, [ssq*2+1] + mov r3d, 0x30 + sub srcq, r6 + kmovb k1, r3d + vpbroadcastq ym2, [srcq+ssq*0] + lea ss3q, [ssq*3] + vpbroadcastq m1, [srcq+ssq*1] + kaddb k2, k1, k1 + vpbroadcastq m2{k1}, [srcq+ssq*2] + add srcq, ss3q + vpbroadcastq m1{k2}, [srcq+ssq*0] ; _ _ 1 3 + punpcklbw m3, m3 + vpbroadcastq m2{k2}, [srcq+ssq*1] ; _ 0 2 4 + psraw m3, 8 ; sign-extend + mova m6, [spel_hv_perm4a] + kshiftrb k1, k1, 2 + movu m7, [spel_hv_perm4b] + pshufb m1, m10 + mova m0, m8 + vpdpbusd m0, m1, m11 + pshufb m2, m10 + mova m1, m8 + vpdpbusd m1, m2, m11 + pshufd m12, m3, q0000 + pshufd m13, m3, q1111 + pshufd m14, m3, q2222 + packssdw m0, m1 ; _ _ _ 0 1 2 3 4 + psraw m0, 2 + vpermb m1, m7, m0 ; 01 12 23 34 +.hv_w4_loop: + movq xm3, [srcq+ssq*2] + movq xm4, [srcq+ss3q ] + lea srcq, [srcq+ssq*4] + vpbroadcastq ym3{k1}, [srcq+ssq*0] ; 5 7 + vpbroadcastq ym4{k1}, [srcq+ssq*1] ; 6 8 + pshufb ym3, ym10 + mova ym2, ym8 + vpdpbusd ym2, ym3, ym11 + pshufb ym4, ym10 + mova ym3, ym8 + vpdpbusd ym3, ym4, ym11 + mova m4, m9 + vpdpwssd m4, m1, m12 ; a0 b0 c0 d0 + packssdw ym2, ym3 ; 5 6 7 8 + psraw ym2, 2 + vshufi32x4 m0, m2, q1032 ; _ 2 3 4 5 6 7 8 + vpermb m2, m6, m0 ; 23 34 45 56 + vpermb m1, m7, m0 ; 45 56 67 78 + vpdpwssd m4, m2, m13 ; a1 b1 c1 d1 + vpdpwssd m4, m1, m14 ; a2 b2 c2 d2 + psrad m4, 6 + vpmovdw [tmpq], m4 + add tmpq, 32 + sub hd, 4 + jg .hv_w4_loop + RET +.hv_w8: + shr mxd, 16 + vpbroadcastd m10, [base+subpel_filters+mxq*8+0] + vpbroadcastd m11, [base+subpel_filters+mxq*8+4] + movzx mxd, myb + shr myd, 16 + cmp hd, 4 + cmove myd, mxd + vpbroadcastq m0, [base+subpel_filters+1+myq*8] + lea r6, [ssq*2+3] + punpcklbw m0, m0 + sub srcq, r6 + psraw m0, 8 ; sign-extend + lea ss3q, [ssq*3] + pshufd m12, m0, q0000 + pshufd m13, m0, q1111 + pshufd m14, m0, q2222 + cmp wd, 8 + jg .hv_w16 + movu xm16, [srcq+ssq*0] + vbroadcasti32x4 m19, [subpel_h_shufA] + vinserti128 ym16, [srcq+ssq*1], 1 + vbroadcasti32x4 m21, [subpel_h_shufC] + vinserti32x4 m16, [srcq+ssq*2], 2 + add srcq, ss3q + vinserti32x4 m16, [srcq+ssq*0], 3 + movu xm17, [srcq+ssq*1] + vbroadcasti32x4 m20, [subpel_h_shufB] + pshufb m3, m16, m19 ; 0 1 2 3 0123 + mova m2, m8 + pshufb m0, m16, m21 ; 0 1 2 3 89ab + vpdpbusd m2, m3, m10 + mova m3, m8 + pshufb xm1, xm17, xm19 ; 3 4 5 6 0123 + vpdpbusd m3, m0, m11 + mova xm0, xm8 + pshufb xm18, xm17, xm21 ; 3 4 5 6 89ab + vpdpbusd xm0, xm1, xm10 + mova xm1, xm8 + pshufb m16, m20 ; 0 1 2 3 4567 + vpdpbusd xm1, xm18, xm11 + pshufb xm17, xm20 ; 3 4 5 6 4567 + vpdpbusd m2, m16, m11 + vpdpbusd m3, m16, m10 + vpdpbusd xm0, xm17, xm11 + vpdpbusd xm1, xm17, xm10 + packssdw m2, m3 + packssdw xm0, xm1 + psraw m2, 2 ; 0 1 2 3 + psraw xm0, 2 ; 4 + valignq m0, m2, 2 ; 1 2 3 4 + punpcklwd m1, m2, m0 ; 01 12 23 34 + punpckhwd m2, m0 +.hv_w8_loop: + movu xm16, [srcq+ssq*2] + vinserti128 ym16, [srcq+ss3q ], 1 + lea srcq, [srcq+ssq*4] + vinserti32x4 m16, [srcq+ssq*0], 2 + vinserti32x4 m16, [srcq+ssq*1], 3 + pshufb m6, m16, m19 ; 5 6 7 8 0123 + mova m5, m8 + pshufb m3, m16, m21 ; 5 6 7 8 89ab + vpdpbusd m5, m6, m10 + mova m6, m8 + pshufb m16, m20 ; 5 6 7 8 4567 + vpdpbusd m6, m3, m11 + mova m3, m9 + vpdpwssd m3, m1, m12 ; a0 b0 c0 d0 + mova m4, m9 + vpdpwssd m4, m2, m12 + vpdpbusd m5, m16, m11 + vpdpbusd m6, m16, m10 + mova m16, m1 + packssdw m5, m6 + mova m6, m2 + psraw m5, 2 ; 5 6 7 8 + valignq m2, m5, m0, 6 ; 4 5 6 7 + mova m0, m5 + punpcklwd m1, m2, m5 ; 45 56 67 78 + punpckhwd m2, m5 + vpdpwssd m3, m1, m14 ; a2 b2 c2 d2 + vpdpwssd m4, m2, m14 + vshufi32x4 m16, m1, q1032 ; 23 34 45 56 + vshufi32x4 m6, m2, q1032 + vpdpwssd m3, m16, m13 ; a1 b1 c1 d1 + vpdpwssd m4, m6, m13 + psrad m3, 6 + psrad m4, 6 + packssdw m3, m4 + mova [tmpq], m3 + add tmpq, 64 + sub hd, 4 + jg .hv_w8_loop + vzeroupper + RET +.hv_w16: + mova m16, [spel_h_perm16] + vpbroadcastd m18, [pb_4] + add wd, wd + paddb m17, m18, m16 + lea r6d, [hq+wq*8-256] + paddb m18, m17 +.hv_w16_loop0: + movu ym19, [srcq+ssq*0] + vinserti32x8 m19, [srcq+ssq*1], 1 + lea r5, [srcq+ssq*2] + movu ym20, [r5 +ssq*0] + vinserti32x8 m20, [r5 +ssq*1], 1 + lea r5, [r5 +ssq*2] + movu ym21, [r5 +ssq*0] + mov r7, tmpq + vpermb m3, m16, m19 ; 0 1 0123 89ab + mova m2, m8 + vpermb m4, m18, m19 ; 0 1 89ab ghij + vpdpbusd m2, m3, m10 + mova m3, m8 + vpermb m5, m16, m20 ; 2 3 0123 89ab + vpdpbusd m3, m4, m11 + mova m4, m8 + vpermb m0, m18, m20 ; 2 3 89ab ghij + vpdpbusd m4, m5, m10 + mova m5, m8 + vpermb ym1, ym16, ym21 ; 4 0123 89ab + vpdpbusd m5, m0, m11 + mova ym0, ym8 + vpermb ym6, ym18, ym21 ; 4 89ab ghij + vpdpbusd ym0, ym1, ym10 + mova ym1, ym8 + vpermb m19, m17, m19 ; 0 1 4567 cdef + vpdpbusd ym1, ym6, ym11 + vpermb m20, m17, m20 ; 2 3 4567 cdef + vpdpbusd m2, m19, m11 + vpdpbusd m3, m19, m10 + vpermb ym21, ym17, ym21 ; 4 4567 cdef + vpdpbusd m4, m20, m11 + vpdpbusd m5, m20, m10 + vpdpbusd ym0, ym21, ym11 + vpdpbusd ym1, ym21, ym10 + packssdw m2, m3 ; 0 1 + packssdw m4, m5 ; 2 3 + packssdw ym0, ym1 ; 4 + REPX {psraw x, 2}, m2, m4, ym0 + vshufi32x4 m3, m2, m4, q1032 ; 1 2 + vshufi32x4 m0, m4, m0, q1032 ; 3 4 + punpcklwd m1, m2, m3 ; 01 12 + punpckhwd m2, m3 + punpcklwd m3, m4, m0 ; 23 34 + punpckhwd m4, m0 +.hv_w16_loop: + movu ym19, [r5+ssq*1] + lea r5, [r5+ssq*2] + vinserti32x8 m19, [r5+ssq*0], 1 + vpermb m6, m16, m19 ; 5 6 0123 89ab + mova m5, m8 + vpermb m20, m18, m19 ; 5 6 89ab ghij + vpdpbusd m5, m6, m10 + mova m6, m8 + vpermb m19, m17, m19 ; 5 6 4567 cdef + vpdpbusd m6, m20, m11 + mova m20, m9 + vpdpwssd m20, m1, m12 ; a0 b0 + mova m21, m9 + vpdpwssd m21, m2, m12 + vpdpbusd m5, m19, m11 + vpdpbusd m6, m19, m10 + vpdpwssd m20, m3, m13 ; a1 b1 + vpdpwssd m21, m4, m13 + packssdw m5, m6 + mova m1, m3 + psraw m5, 2 ; 5 6 + mova m2, m4 + vshufi32x4 m4, m0, m5, q1032 ; 4 5 + mova m0, m5 + punpcklwd m3, m4, m0 ; 45 56 + punpckhwd m4, m0 + vpdpwssd m20, m3, m14 ; a2 b2 + vpdpwssd m21, m4, m14 + psrad m20, 6 + psrad m21, 6 + packssdw m20, m21 + mova [r7+wq*0], ym20 + vextracti32x8 [r7+wq*1], m20, 1 + lea r7, [r7+wq*2] + sub hd, 2 + jg .hv_w16_loop + add srcq, 16 + add tmpq, 32 movzx hd, r6b - mov srcq, r4 - mov dstq, r7 sub r6d, 1<<8 jg .hv_w16_loop0 vzeroupper @@ -2353,183 +3424,38 @@ cglobal put_8tap_8bpc, 4, 9, 0, dst, ds, src, ss, w, h, mx, my, ss3 mova [tmpq+64*1], m1 %endmacro -%if WIN64 -DECLARE_REG_TMP 6, 4 -%else -DECLARE_REG_TMP 6, 7 -%endif - -%define PREP_8TAP_FN FN prep_8tap, - +PREP_8TAP_FN smooth_sharp, SMOOTH, SHARP, prep_8tap_8bpc +PREP_8TAP_FN regular_sharp, REGULAR, SHARP, prep_8tap_8bpc PREP_8TAP_FN sharp, SHARP, SHARP -PREP_8TAP_FN sharp_smooth, SHARP, SMOOTH -PREP_8TAP_FN smooth_sharp, SMOOTH, SHARP -PREP_8TAP_FN smooth, SMOOTH, SMOOTH -PREP_8TAP_FN sharp_regular, SHARP, REGULAR -PREP_8TAP_FN regular_sharp, REGULAR, SHARP -PREP_8TAP_FN smooth_regular, SMOOTH, REGULAR -PREP_8TAP_FN regular_smooth, REGULAR, SMOOTH -PREP_8TAP_FN regular, REGULAR, REGULAR -cglobal prep_8tap_8bpc, 3, 8, 0, tmp, src, stride, w, h, mx, my, stride3 +cglobal prep_8tap_8bpc, 4, 8, 0, tmp, src, stride, w, h, mx, my, stride3 imul mxd, mxm, 0x010101 add mxd, t0d ; 8tap_h, mx, 4tap_h imul myd, mym, 0x010101 add myd, t1d ; 8tap_v, my, 4tap_v lea r7, [prep_avx512icl] - movsxd wq, wm movifnidn hd, hm test mxd, 0xf00 jnz .h test myd, 0xf00 - jnz .v - tzcnt wd, wd - movzx wd, word [r7+wq*2+table_offset(prep,)] - add wq, r7 - lea r6, [strideq*3] -%if WIN64 - pop r7 -%endif - jmp wq -.h: - test myd, 0xf00 - jnz .hv - vpbroadcastd m4, [pd_2] - WIN64_SPILL_XMM 10 - cmp wd, 4 - je .h_w4 - tzcnt wd, wd - shr mxd, 16 - sub srcq, 3 - movzx wd, word [r7+wq*2+table_offset(prep, _8tap_h)] - vpbroadcastd m8, [r7+mxq*8+subpel_filters-prep_avx512icl+0] - vpbroadcastd m9, [r7+mxq*8+subpel_filters-prep_avx512icl+4] - add wq, r7 - jmp wq -.h_w4: - movzx mxd, mxb - vbroadcasti128 ym5, [subpel_h_shufA] - mov r3d, 0x4 - dec srcq - vpbroadcastd ym6, [r7+mxq*8+subpel_filters-prep_avx512icl+2] - kmovb k1, r3d - lea stride3q, [strideq*3] -.h_w4_loop: - movq xm2, [srcq+strideq*0] - movq xm3, [srcq+strideq*1] - vpbroadcastq ym2{k1}, [srcq+strideq*2] - vpbroadcastq ym3{k1}, [srcq+stride3q ] - lea srcq, [srcq+strideq*4] - pshufb ym2, ym5 - pshufb ym3, ym5 - mova ym0, ym4 - vpdpbusd ym0, ym2, ym6 - mova ym1, ym4 - vpdpbusd ym1, ym3, ym6 - packssdw ym0, ym1 - psraw ym0, 2 - mova [tmpq], ym0 - add tmpq, 32 - sub hd, 4 - jg .h_w4_loop - RET -.h_w8: - vbroadcasti128 m5, [subpel_h_shufA] - vbroadcasti128 m6, [subpel_h_shufB] - vbroadcasti128 m7, [subpel_h_shufC] - lea stride3q, [strideq*3] -.h_w8_loop: - movu xmm3, [srcq+strideq*0] - vinserti128 ym3, ymm3, [srcq+strideq*1], 1 - vinserti128 m3, [srcq+strideq*2], 2 - vinserti128 m3, [srcq+stride3q ], 3 - lea srcq, [srcq+strideq*4] - pshufb m1, m3, m5 - pshufb m2, m3, m6 - mova m0, m4 - vpdpbusd m0, m1, m8 - mova m1, m4 - vpdpbusd m1, m2, m8 - pshufb m3, m7 - vpdpbusd m0, m2, m9 - vpdpbusd m1, m3, m9 - packssdw m0, m1 - psraw m0, 2 - mova [tmpq], m0 - add tmpq, 64 - sub hd, 4 - jg .h_w8_loop - RET -.h_w16: - mova m5, [spel_h_perm16a] - mova m6, [spel_h_perm16b] - mova m7, [spel_h_perm16c] - lea stride3q, [strideq*3] -.h_w16_loop: - movu ym0, [srcq+strideq*0] - movu ym1, [srcq+strideq*2] - vinserti32x8 m0, [srcq+strideq*1], 1 - vinserti32x8 m1, [srcq+stride3q ], 1 - lea srcq, [srcq+strideq*4] - PREP_8TAP_H - add tmpq, 64*2 - sub hd, 4 - jg .h_w16_loop - RET -.h_w32: - mova m5, [spel_h_perm32a] - mova m6, [spel_h_perm32b] - mova m7, [spel_h_perm32c] -.h_w32_loop: - movu m0, [srcq+strideq*0] - movu m1, [srcq+strideq*1] - lea srcq, [srcq+strideq*2] - PREP_8TAP_H - add tmpq, 64*2 - sub hd, 2 - jg .h_w32_loop - RET -.h_w64: - xor r6d, r6d - jmp .h_start -.h_w128: - mov r6, -64*1 -.h_start: - mova m5, [spel_h_perm32a] - mova m6, [spel_h_perm32b] - mova m7, [spel_h_perm32c] - sub srcq, r6 - mov r5, r6 -.h_loop: - movu m0, [srcq+r6+32*0] - movu m1, [srcq+r6+32*1] - PREP_8TAP_H - add tmpq, 64*2 - add r6, 64 - jle .h_loop - add srcq, strideq - mov r6, r5 - dec hd - jg .h_loop - RET + jz mangle(private_prefix %+ _prep_6tap_8bpc_avx512icl).prep .v: movzx mxd, myb ; Select 4-tap/8-tap filter multipliers. shr myd, 16 ; Note that the code is 8-tap only, having - tzcnt wd, wd cmp hd, 4 ; a separate 4-tap code path for (4|8|16)x4 cmove myd, mxd ; had a negligible effect on performance. - ; TODO: Would a 6-tap code path be worth it? - lea myq, [r7+myq*8+subpel_filters-prep_avx512icl] - movzx wd, word [r7+wq*2+table_offset(prep, _8tap_v)] - add wq, r7 - lea stride3q, [strideq*3] - sub srcq, stride3q + tzcnt r5d, wd + lea myq, [base+subpel_filters+myq*8] + movzx r5d, word [r7+r5*2+table_offset(prep, _8tap_v)] vpbroadcastd m7, [pw_8192] vpbroadcastw m8, [myq+0] + add r5, r7 vpbroadcastw m9, [myq+2] + lea stride3q, [strideq*3] vpbroadcastw m10, [myq+4] + sub srcq, stride3q vpbroadcastw m11, [myq+6] - jmp wq + jmp r5 .v_w4: movd xmm0, [srcq+strideq*0] vpbroadcastd ymm1, [srcq+strideq*2] @@ -2576,172 +3502,146 @@ cglobal prep_8tap_8bpc, 3, 8, 0, tmp, src, stride, w, h, mx, my, stride3 vzeroupper RET .v_w8: - mov r3d, 0xf044 - kmovw k1, r3d - kshiftrw k2, k1, 8 - movq xm0, [srcq+strideq*0] - vpbroadcastq ym1, [srcq+strideq*1] - vpbroadcastq m2, [srcq+strideq*2] - vpbroadcastq m3, [srcq+stride3q ] + mova m6, [spel_v_perm8] + movq xm1, [srcq+strideq*0] + mov r6d, 0x3e + movq xm2, [srcq+strideq*1] + vpbroadcastq ym3, [srcq+strideq*2] + kmovb k1, r6d + vpbroadcastq ym4, [srcq+stride3q ] lea srcq, [srcq+strideq*4] - vpbroadcastq m4, [srcq+strideq*0] - vpbroadcastq m5, [srcq+strideq*1] - vpbroadcastq m6, [srcq+strideq*2] - vmovdqa64 ym0{k1}, ym1 - vmovdqa64 ym1{k1}, ym2 - vmovdqa64 m2{k1}, m3 - vmovdqa64 m3{k1}, m4 - vmovdqa64 m4{k1}, m5 - vmovdqa64 m5{k1}, m6 - punpcklbw ym0, ym1 ; 01 12 __ __ - punpcklbw m2, m3 ; 23 34 23 34 - punpcklbw m4, m5 ; 45 56 45 56 - vmovdqa64 m0{k2}, m2 ; 01 12 23 34 - vmovdqa64 m2{k2}, m4 ; 23 34 45 56 + vpunpcklqdq m1{k1}, m3, [srcq+strideq*0] {1to8} + vpunpcklqdq m2{k1}, m4, [srcq+strideq*1] {1to8} + movq xm0, [srcq+strideq*2] + kshiftlb k2, k1, 2 + shufpd m1, m2, 0x30 ; 0 1 2 3 4 5 + vshufi32x4 m2, m1, m0, q0021 ; 2 3 4 5 6 _ + vpermb m1, m6, m1 ; 01 12 23 34 + vpermb m2, m6, m2 ; 23 34 45 56 .v_w8_loop: - vpbroadcastq m1, [srcq+stride3q ] + vpbroadcastq ym3, [srcq+strideq*4] + vpunpcklqdq ym0{k1}, ym3, [srcq+stride3q] {1to4} lea srcq, [srcq+strideq*4] - vpbroadcastq m3, [srcq+strideq*0] - vpbroadcastq m5, [srcq+strideq*1] - pmaddubsw m14, m0, m8 - pmaddubsw m15, m2, m9 - vpblendmq m0{k1}, m6, m1 - vpblendmq m2{k1}, m1, m3 - vpbroadcastq m6, [srcq+strideq*2] - paddw m14, m15 - punpcklbw m2, m0, m2 ; 67 78 67 78 - vpblendmq m12{k1}, m3, m5 - vpblendmq m13{k1}, m5, m6 - vpblendmq m0{k2}, m4, m2 ; 45 56 67 78 - punpcklbw m4, m12, m13 ; 89 9a 89 9a - vmovdqa64 m2{k2}, m4 ; 67 78 89 9a - pmaddubsw m12, m0, m10 - pmaddubsw m13, m2, m11 - paddw m14, m12 - paddw m14, m13 - pmulhrsw m14, m7 - mova [tmpq], m14 + vpbroadcastq m3, [srcq+strideq*2] + vpunpcklqdq m0{k2}, m3, [srcq+strideq*1] {1to8} + pmaddubsw m4, m1, m8 ; a0 b0 c0 d0 + mova m1, m2 + pmaddubsw m5, m2, m9 ; a1 b1 c1 d1 + vpermb m2, m6, m0 ; 67 78 89 9a + mova xm0, xm3 + vshufi32x4 m1, m2, q1032 ; 45 56 67 78 + pmaddubsw m3, m2, m11 ; a3 b3 c3 d3 + paddw m4, m5 + pmaddubsw m5, m1, m10 ; a2 b2 c2 d2 + paddw m4, m3 + paddw m4, m5 + pmulhrsw m4, m7 + mova [tmpq], m4 add tmpq, 64 sub hd, 4 jg .v_w8_loop RET .v_w16: - mov r3d, 0xf0 - kmovb k1, r3d - vbroadcasti128 m0, [srcq+strideq*0] - vbroadcasti128 m1, [srcq+strideq*1] - vbroadcasti128 m2, [srcq+strideq*2] - vbroadcasti128 m3, [srcq+stride3q ] + mova m12, [spel_v_perm16b] + vbroadcasti32x4 m1, [srcq+strideq*0] + mov r6d, 0x0f + vbroadcasti32x4 ym4, [srcq+strideq*1] + vbroadcasti32x4 m2, [srcq+strideq*2] + kmovb k1, r6d + vbroadcasti32x4 ym5, [srcq+stride3q ] lea srcq, [srcq+strideq*4] - vbroadcasti128 m4, [srcq+strideq*0] - vbroadcasti128 m5, [srcq+strideq*1] - vbroadcasti128 m6, [srcq+strideq*2] - vmovdqa64 m0{k1}, m1 - vmovdqa64 m1{k1}, m2 - vmovdqa64 m2{k1}, m3 - vmovdqa64 m3{k1}, m4 - vmovdqa64 m4{k1}, m5 - vmovdqa64 m5{k1}, m6 - shufpd m0, m2, 0xcc ; 0a_2a 0b_2b 1a_3a 1b_3b - shufpd m1, m3, 0xcc ; 1a_3a 1b_3b 2a_4a 2b_4b - shufpd m4, m4, 0x44 ; 4a_-- 4b_-- 5a_-- 5b_-- - shufpd m5, m5, 0x44 ; 5a_-- 5b_-- 6a_-- 6b_-- - punpckhbw m2, m0, m1 ; 23a 23b 34a 34b - punpcklbw m0, m1 ; 01a 01b 12a 12b - punpcklbw m4, m5 ; 45a 45b 56a 56b + vbroadcasti32x4 m3, [srcq+strideq*0] + vbroadcasti32x4 ym6, [srcq+strideq*1] + vbroadcasti32x4 m0, [srcq+strideq*2] + vshufpd m1{k1}, m4, m2, 0xcc + vshufpd m2{k1}, m5, m3, 0xcc + vshufpd m3{k1}, m6, m0, 0xcc + vpermb m1, m12, m1 ; 01 12 + vpermb m2, m12, m2 ; 23 34 + vpermb m3, m12, m3 ; 45 56 .v_w16_loop: - vbroadcasti128 m3, [srcq+stride3q ] + pmaddubsw m4, m1, m8 ; a0 b0 + mova m1, m3 + pmaddubsw m13, m2, m9 ; a1 b1 + vbroadcasti32x4 ym6, [srcq+stride3q ] + pmaddubsw m5, m2, m8 ; c0 d0 lea srcq, [srcq+strideq*4] - vbroadcasti128 m5, [srcq+strideq*0] - vpblendmq m1{k1}, m6, m3 - vmovdqa64 m3{k1}, m5 - pmaddubsw m12, m0, m8 - pmaddubsw m13, m2, m8 - pmaddubsw m14, m2, m9 - pmaddubsw m15, m4, m9 - pmaddubsw m0, m4, m10 - vbroadcasti128 m2, [srcq+strideq*1] - vbroadcasti128 m6, [srcq+strideq*2] - paddw m12, m14 - paddw m13, m15 - paddw m12, m0 - vmovdqa64 m5{k1}, m2 - vmovdqa64 m2{k1}, m6 - mova m0, m4 - shufpd m1, m5, 0xcc ; 6a_8a 6b_8b 7a_9a 7b_9b - shufpd m3, m2, 0xcc ; 7a_9a 7b_9b 8a_Aa 8b_Ab - punpcklbw m2, m1, m3 ; 67a 67b 78a 78b - punpckhbw m4, m1, m3 ; 89a 89b 9Aa 9Ab - pmaddubsw m14, m2, m10 - pmaddubsw m15, m2, m11 - paddw m13, m14 - paddw m12, m15 - pmaddubsw m14, m4, m11 - paddw m13, m14 - pmulhrsw m12, m7 - pmulhrsw m13, m7 - mova [tmpq+ 0], m12 - mova [tmpq+64], m13 + pmaddubsw m14, m3, m9 ; c1 d1 + vbroadcasti32x4 m3, [srcq+strideq*0] + vshufpd m0{k1}, m6, m3, 0xcc + vbroadcasti32x4 ym6, [srcq+strideq*1] + vpermb m2, m12, m0 ; 67 78 + vbroadcasti32x4 m0, [srcq+strideq*2] + vshufpd m3{k1}, m6, m0, 0xcc + paddw m4, m13 + pmaddubsw m13, m1, m10 ; a2 b2 + vpermb m3, m12, m3 ; 89 9a + paddw m5, m14 + pmaddubsw m14, m2, m10 ; c2 d2 + pmaddubsw m15, m2, m11 ; a3 b3 + pmaddubsw m6, m3, m11 ; c3 d3 + paddw m4, m13 + paddw m5, m14 + paddw m4, m15 + paddw m5, m6 + pmulhrsw m4, m7 + pmulhrsw m5, m7 + mova [tmpq+ 0], m4 + mova [tmpq+64], m5 add tmpq, 64*2 sub hd, 4 jg .v_w16_loop RET .v_w32: - mova m18, [bilin_v_perm64] - movu ym0, [srcq+strideq*0] - movu ym1, [srcq+strideq*1] - lea srcq, [srcq+strideq*2] - movu ym2, [srcq+strideq*0] - movu ym3, [srcq+strideq*1] - lea srcq, [srcq+strideq*2] - movu ym4, [srcq+strideq*0] - movu ym5, [srcq+strideq*1] - lea srcq, [srcq+strideq*2] - movu ym6, [srcq+strideq*0] - vpermq m0, m18, m0 - vpermq m1, m18, m1 - vpermq m2, m18, m2 - vpermq m3, m18, m3 - vpermq m4, m18, m4 - vpermq m5, m18, m5 - vpermq m6, m18, m6 - punpcklbw m0, m1 - punpcklbw m1, m2 - punpcklbw m2, m3 - punpcklbw m3, m4 - punpcklbw m4, m5 - punpcklbw m5, m6 + movshdup m21, [bilin_v_perm64] + movu ym16, [srcq+strideq*0] + movu ym17, [srcq+strideq*1] + movu ym18, [srcq+strideq*2] + add srcq, stride3q + movu ym19, [srcq+strideq*0] + vpermt2q m16, m21, m19 ; 0 3 + movu ym20, [srcq+strideq*1] + vpermt2q m17, m21, m20 ; 1 4 + movu ym20, [srcq+strideq*2] + add srcq, stride3q + vpermt2q m18, m21, m20 ; 2 5 + movu ym20, [srcq+strideq*0] + vpermt2q m19, m21, m20 ; 3 6 + punpcklbw m0, m16, m17 ; 01 + punpcklbw m1, m17, m18 ; 12 + punpcklbw m2, m18, m19 ; 23 + punpckhbw m3, m16, m17 ; 34 + punpckhbw m4, m17, m18 ; 45 + punpckhbw m5, m18, m19 ; 56 .v_w32_loop: - movu ym12, [srcq+strideq*1] + movu ym16, [srcq+strideq*1] lea srcq, [srcq+strideq*2] - movu ym13, [srcq+strideq*0] + movu ym17, [srcq+strideq*0] pmaddubsw m14, m0, m8 - pmaddubsw m16, m2, m9 - pmaddubsw m15, m1, m8 - pmaddubsw m17, m3, m9 mova m0, m2 + pmaddubsw m15, m1, m8 mova m1, m3 - vpermq m12, m18, m12 - vpermq m13, m18, m13 - paddw m14, m16 - paddw m15, m17 - pmaddubsw m16, m4, m10 - pmaddubsw m17, m5, m10 - punpcklbw m6, m12 - punpcklbw m12, m13 + pmaddubsw m2, m9 + vpermt2q m16, m21, m17 ; 7 8 + pmaddubsw m3, m9 + pmaddubsw m12, m4, m10 + pmaddubsw m13, m5, m10 + shufpd m19, m16, 0x55 ; 6 7 + paddw m14, m2 mova m2, m4 + punpcklbw m4, m19, m16 ; 67 + paddw m15, m3 mova m3, m5 - paddw m14, m16 - paddw m15, m17 - pmaddubsw m16, m6, m11 - pmaddubsw m17, m12, m11 - mova m4, m6 - mova m5, m12 - paddw m14, m16 - paddw m15, m17 + punpckhbw m5, m19, m16 ; 78 + paddw m14, m12 + paddw m15, m13 + pmaddubsw m12, m4, m11 + pmaddubsw m13, m5, m11 + mova m19, m16 + paddw m14, m12 + paddw m15, m13 pmulhrsw m14, m7 pmulhrsw m15, m7 - mova m6, m13 mova [tmpq+ 0], m14 mova [tmpq+64], m15 add tmpq, 64*2 @@ -2750,154 +3650,241 @@ cglobal prep_8tap_8bpc, 3, 8, 0, tmp, src, stride, w, h, mx, my, stride3 vzeroupper RET .v_w64: - mov wd, 64 - jmp .v_start .v_w128: - mov wd, 128 -.v_start: - WIN64_SPILL_XMM 27 - mova m26, [bilin_v_perm64] - lea r6d, [hq+wq*2] - mov r5, srcq - mov r7, tmpq + WIN64_SPILL_XMM 24 + mova m23, [bilin_v_perm64] + add wd, wd + lea r6d, [hq+wq] .v_loop0: - vpermq m0, m26, [srcq+strideq*0] - vpermq m1, m26, [srcq+strideq*1] - lea srcq, [srcq+strideq*2] - vpermq m2, m26, [srcq+strideq*0] - vpermq m3, m26, [srcq+strideq*1] - lea srcq, [srcq+strideq*2] - vpermq m4, m26, [srcq+strideq*0] - vpermq m5, m26, [srcq+strideq*1] - lea srcq, [srcq+strideq*2] - vpermq m6, m26, [srcq+strideq*0] - punpckhbw m12, m0, m1 - punpcklbw m0, m1 - punpckhbw m13, m1, m2 - punpcklbw m1, m2 - punpckhbw m14, m2, m3 - punpcklbw m2, m3 - punpckhbw m15, m3, m4 - punpcklbw m3, m4 - punpckhbw m16, m4, m5 - punpcklbw m4, m5 - punpckhbw m17, m5, m6 - punpcklbw m5, m6 + vpermq m12, m23, [srcq+strideq*0] + vpermq m13, m23, [srcq+strideq*1] + lea r5, [srcq+strideq*2] + vpermq m14, m23, [r5 +strideq*0] + vpermq m15, m23, [r5 +strideq*1] + lea r5, [r5+strideq*2] + vpermq m16, m23, [r5 +strideq*0] + vpermq m17, m23, [r5 +strideq*1] + lea r5, [r5+strideq*2] + vpermq m18, m23, [r5 +strideq*0] + mov r7, tmpq + punpcklbw m0, m12, m13 ; 01 + punpckhbw m12, m13 + punpcklbw m1, m13, m14 ; 12 + punpckhbw m13, m14 + punpcklbw m2, m14, m15 ; 23 + punpckhbw m14, m15 + punpcklbw m3, m15, m16 ; 34 + punpckhbw m15, m16 + punpcklbw m4, m16, m17 ; 45 + punpckhbw m16, m17 + punpcklbw m5, m17, m18 ; 56 + punpckhbw m17, m18 .v_loop: - vpermq m18, m26, [srcq+strideq*1] - lea srcq, [srcq+strideq*2] - vpermq m19, m26, [srcq+strideq*0] - pmaddubsw m20, m0, m8 - pmaddubsw m21, m12, m8 - pmaddubsw m22, m1, m8 - pmaddubsw m23, m13, m8 + pmaddubsw m19, m0, m8 ; a0 + vpermq m6, m23, [r5+strideq*1] + pmaddubsw m20, m12, m8 mova m0, m2 + pmaddubsw m2, m9 ; a1 mova m12, m14 + pmaddubsw m14, m9 + lea r5, [r5+strideq*2] + pmaddubsw m21, m1, m8 ; b0 + pmaddubsw m22, m13, m8 mova m1, m3 + pmaddubsw m3, m9 ; b1 mova m13, m15 - pmaddubsw m2, m9 - pmaddubsw m14, m9 - pmaddubsw m3, m9 pmaddubsw m15, m9 - punpckhbw m24, m6, m18 - punpcklbw m6, m18 - paddw m20, m2 - paddw m21, m14 - paddw m22, m3 - paddw m23, m15 + paddw m19, m2 mova m2, m4 + pmaddubsw m4, m10 ; a2 + paddw m20, m14 mova m14, m16 + pmaddubsw m16, m10 + paddw m21, m3 mova m3, m5 + pmaddubsw m5, m10 ; b2 + paddw m22, m15 mova m15, m17 - pmaddubsw m4, m10 - pmaddubsw m16, m10 - pmaddubsw m5, m10 pmaddubsw m17, m10 - punpckhbw m25, m18, m19 - punpcklbw m18, m19 - paddw m20, m4 - paddw m21, m16 - paddw m22, m5 - paddw m23, m17 - mova m4, m6 - mova m16, m24 - mova m5, m18 - mova m17, m25 - pmaddubsw m6, m11 - pmaddubsw m24, m11 - pmaddubsw m18, m11 - pmaddubsw m25, m11 - paddw m20, m6 - paddw m21, m24 - paddw m22, m18 - paddw m23, m25 - pmulhrsw m20, m7 - pmulhrsw m21, m7 - pmulhrsw m22, m7 - pmulhrsw m23, m7 - mova m6, m19 - mova [tmpq+wq*0+ 0], m20 - mova [tmpq+wq*0+64], m21 - mova [tmpq+wq*2+ 0], m22 - mova [tmpq+wq*2+64], m23 - lea tmpq, [tmpq+wq*4] + paddw m19, m4 + punpcklbw m4, m18, m6 ; 67 + paddw m20, m16 + punpckhbw m16, m18, m6 + vpermq m18, m23, [r5+strideq*0] + paddw m21, m5 + pmaddubsw m5, m4, m11 ; a3 + paddw m22, m17 + pmaddubsw m17, m16, m11 + paddw m19, m5 + punpcklbw m5, m6, m18 ; 78 + paddw m20, m17 + punpckhbw m17, m6, m18 + pmaddubsw m6, m5, m11 ; b3 + paddw m21, m6 + pmaddubsw m6, m17, m11 + paddw m22, m6 + REPX {pmulhrsw x, m7}, m19, m20, m21, m22 + mova [r7+wq*0+ 0], m19 + mova [r7+wq*0+64], m20 + mova [r7+wq*1+ 0], m21 + mova [r7+wq*1+64], m22 + lea r7, [r7+wq*2] sub hd, 2 jg .v_loop - add r5, 64 - add r7, 128 + add srcq, 64 + add tmpq, 128 movzx hd, r6b - mov srcq, r5 - mov tmpq, r7 sub r6d, 1<<8 jg .v_loop0 RET -.hv: - WIN64_SPILL_XMM 16 +.h: + RESET_STACK_STATE + test myd, 0xf00 + jnz .hv +.h2: + vpbroadcastd m4, [pd_2] cmp wd, 4 - je .hv_w4 + je .h_w4 + tzcnt wd, wd shr mxd, 16 sub srcq, 3 - vpbroadcastd m10, [r7+mxq*8+subpel_filters-prep_avx512icl+0] - vpbroadcastd m11, [r7+mxq*8+subpel_filters-prep_avx512icl+4] - movzx mxd, myb - shr myd, 16 - cmp hd, 4 - cmove myd, mxd - tzcnt wd, wd - vpbroadcastd m8, [pd_2] - movzx wd, word [r7+wq*2+table_offset(prep, _8tap_hv)] - vpbroadcastd m9, [pd_32] + movzx wd, word [r7+wq*2+table_offset(prep, _8tap_h)] + vpbroadcastd m8, [base+subpel_filters+mxq*8+0] + vpbroadcastd m9, [base+subpel_filters+mxq*8+4] add wq, r7 - vpbroadcastq m0, [r7+myq*8+subpel_filters-prep_avx512icl] - lea stride3q, [strideq*3] - sub srcq, stride3q - punpcklbw m0, m0 - psraw m0, 8 ; sign-extend - pshufd m12, m0, q0000 - pshufd m13, m0, q1111 - pshufd m14, m0, q2222 - pshufd m15, m0, q3333 jmp wq -.hv_w4: +.h_w4: + movzx mxd, mxb + vbroadcasti128 ym5, [subpel_h_shufA] + mov r3d, 0x4 + dec srcq + vpbroadcastd ym6, [base+subpel_filters+mxq*8+2] + kmovb k1, r3d + lea stride3q, [strideq*3] +.h_w4_loop: + movq xm2, [srcq+strideq*0] + movq xm3, [srcq+strideq*1] + vpbroadcastq ym2{k1}, [srcq+strideq*2] + vpbroadcastq ym3{k1}, [srcq+stride3q ] + lea srcq, [srcq+strideq*4] + pshufb ym2, ym5 + pshufb ym3, ym5 + mova ym0, ym4 + vpdpbusd ym0, ym2, ym6 + mova ym1, ym4 + vpdpbusd ym1, ym3, ym6 + packssdw ym0, ym1 + psraw ym0, 2 + mova [tmpq], ym0 + add tmpq, 32 + sub hd, 4 + jg .h_w4_loop + RET +.h_w8: + vbroadcasti128 m5, [subpel_h_shufA] + vbroadcasti128 m6, [subpel_h_shufB] + vbroadcasti128 m7, [subpel_h_shufC] + lea stride3q, [strideq*3] +.h_w8_loop: + movu xmm3, [srcq+strideq*0] + vinserti128 ym3, ymm3, [srcq+strideq*1], 1 + vinserti128 m3, [srcq+strideq*2], 2 + vinserti128 m3, [srcq+stride3q ], 3 + lea srcq, [srcq+strideq*4] + pshufb m1, m3, m5 + pshufb m2, m3, m6 + mova m0, m4 + vpdpbusd m0, m1, m8 + mova m1, m4 + vpdpbusd m1, m2, m8 + pshufb m3, m7 + vpdpbusd m0, m2, m9 + vpdpbusd m1, m3, m9 + packssdw m0, m1 + psraw m0, 2 + mova [tmpq], m0 + add tmpq, 64 + sub hd, 4 + jg .h_w8_loop + RET +.h_w16: + mova m5, [spel_h_perm16] + vpbroadcastd m7, [pb_4] + lea stride3q, [strideq*3] + paddb m6, m7, m5 + paddb m7, m6 +.h_w16_loop: + movu ym0, [srcq+strideq*0] + movu ym1, [srcq+strideq*2] + vinserti32x8 m0, [srcq+strideq*1], 1 + vinserti32x8 m1, [srcq+stride3q ], 1 + lea srcq, [srcq+strideq*4] + PREP_8TAP_H + add tmpq, 64*2 + sub hd, 4 + jg .h_w16_loop + RET +.h_w32: + mova m5, [spel_h_perm32] + vpbroadcastd m7, [pb_4] + paddb m6, m7, m5 + paddb m7, m6 +.h_w32_loop: + movu m0, [srcq+strideq*0] + movu m1, [srcq+strideq*1] + lea srcq, [srcq+strideq*2] + PREP_8TAP_H + add tmpq, 64*2 + sub hd, 2 + jg .h_w32_loop + RET +.h_w64: + xor r6d, r6d + jmp .h_start +.h_w128: + mov r6, -64*1 +.h_start: + mova m5, [spel_h_perm32] + vpbroadcastd m7, [pb_4] + sub srcq, r6 + paddb m6, m7, m5 + paddb m7, m6 +.h_loop0: + mov r5, r6 +.h_loop: + movu m0, [srcq+r5+32*0] + movu m1, [srcq+r5+32*1] + PREP_8TAP_H + add tmpq, 64*2 + add r5, 64 + jle .h_loop + add srcq, strideq + dec hd + jg .h_loop0 + RET +.hv: + RESET_STACK_STATE + vpbroadcastd m8, [pd_2] + vpbroadcastd m9, [pd_32] + cmp wd, 4 + jg .hv_w8 movzx mxd, mxb dec srcq - vpbroadcastd m8, [r7+mxq*8+subpel_filters-prep_avx512icl+2] + vpbroadcastd m11, [base+subpel_filters+mxq*8+2] movzx mxd, myb shr myd, 16 cmp hd, 4 cmove myd, mxd - vpbroadcastq m0, [r7+myq*8+subpel_filters-prep_avx512icl] + vpbroadcastq m0, [base+subpel_filters+myq*8] lea stride3q, [strideq*3] sub srcq, stride3q mov r3d, 0x04 kmovb k1, r3d kshiftlb k2, k1, 2 kshiftlb k3, k1, 4 - vpbroadcastd m10, [pd_2] - vbroadcasti128 m16, [subpel_h_shufA] + vbroadcasti128 m10, [subpel_h_shufA] punpcklbw m0, m0 psraw m0, 8 ; sign-extend - vpbroadcastd m11, [pd_32] pshufd m12, m0, q0000 pshufd m13, m0, q1111 pshufd m14, m0, q2222 @@ -2910,263 +3897,265 @@ cglobal prep_8tap_8bpc, 3, 8, 0, tmp, src, stride, w, h, mx, my, stride3 vpbroadcastq m3{k2}, [srcq+strideq*0] vpbroadcastq m2{k3}, [srcq+strideq*1] vpbroadcastq m3{k3}, [srcq+strideq*2] - mova m17, [spel_hv_perm4a] - movu m18, [spel_hv_perm4b] - mova m0, m10 - mova m1, m10 - pshufb m2, m16 - pshufb m3, m16 - vpdpbusd m0, m2, m8 - vpdpbusd m1, m3, m8 + mova m6, [spel_hv_perm4a] + movu m7, [spel_hv_perm4b] + mova m0, m8 + mova m1, m8 + pshufb m2, m10 + pshufb m3, m10 + vpdpbusd m0, m2, m11 + vpdpbusd m1, m3, m11 packssdw m0, m1 ; _ 0 1 2 3 4 5 6 psraw m0, 2 - vpermb m1, m17, m0 ; 01 12 23 34 - vpermb m2, m18, m0 ; 23 34 45 56 + vpermb m1, m6, m0 ; 01 12 23 34 + vpermb m2, m7, m0 ; 23 34 45 56 .hv_w4_loop: movq xm3, [srcq+stride3q ] lea srcq, [srcq+strideq*4] movq xm4, [srcq+strideq*0] vpbroadcastq ym3{k1}, [srcq+strideq*1] vpbroadcastq ym4{k1}, [srcq+strideq*2] - mova ym5, ym10 - mova ym6, ym10 - pshufb ym3, ym16 - pshufb ym4, ym16 - vpdpbusd ym5, ym3, ym8 - vpdpbusd ym6, ym4, ym8 - mova m7, m11 - packssdw ym5, ym6 ; 7 8 9 a _ _ _ _ - psraw ym5, 2 - valignq m0, m5, m0, 4 ; _ 4 5 6 7 8 9 a - vpdpwssd m7, m1, m12 - vpdpwssd m7, m2, m13 - vpermb m1, m17, m0 ; 45 56 67 78 - vpermb m2, m18, m0 ; 67 78 89 9a - vpdpwssd m7, m1, m14 - vpdpwssd m7, m2, m15 - psrad m7, 6 - vpmovdw [tmpq], m7 + mova m5, m9 + pshufb ym3, ym10 + vpdpwssd m5, m1, m12 ; a0 b0 c0 d0 + mova ym1, ym8 + pshufb ym4, ym10 + vpdpbusd ym1, ym3, ym11 + mova ym3, ym8 + vpdpbusd ym3, ym4, ym11 + vpdpwssd m5, m2, m13 ; a1 b1 c1 d1 + packssdw ym1, ym3 ; 7 8 9 a + psraw ym1, 2 + vshufi32x4 m0, m1, q1032 ; _ 4 5 6 7 8 9 a + vpermb m1, m6, m0 ; 45 56 67 78 + vpermb m2, m7, m0 ; 67 78 89 9a + vpdpwssd m5, m1, m14 ; a2 b2 c2 d2 + vpdpwssd m5, m2, m15 ; a3 b3 c3 d3 + psrad m5, 6 + vpmovdw [tmpq], m5 add tmpq, 32 sub hd, 4 jg .hv_w4_loop - vzeroupper RET .hv_w8: - WIN64_SPILL_XMM 24 - vbroadcasti128 m16, [subpel_h_shufA] - vbroadcasti128 m17, [subpel_h_shufB] - vbroadcasti128 m18, [subpel_h_shufC] - vinserti128 ym0, [srcq+strideq*0], 1 - vinserti128 m0, [srcq+strideq*1], 2 - vinserti128 m0, [srcq+strideq*2], 3 - movu xm1, [srcq+stride3q ] + shr mxd, 16 + sub srcq, 3 + vpbroadcastd m10, [base+subpel_filters+mxq*8+0] + vpbroadcastd m11, [base+subpel_filters+mxq*8+4] + movzx mxd, myb + shr myd, 16 + cmp hd, 4 + cmove myd, mxd + vpbroadcastq m0, [base+subpel_filters+myq*8] + lea stride3q, [strideq*3] + sub srcq, stride3q + punpcklbw m0, m0 + psraw m0, 8 ; sign-extend + pshufd m12, m0, q0000 + pshufd m13, m0, q1111 + pshufd m14, m0, q2222 + pshufd m15, m0, q3333 + cmp wd, 8 + jg .hv_w16 + vbroadcasti32x4 m17, [srcq+stride3q ] + vinserti32x4 m16, m17, [srcq+strideq*0], 0 + vbroadcasti32x4 m19, [subpel_h_shufA] + vinserti32x4 m16, [srcq+strideq*1], 1 + vbroadcasti32x4 m21, [subpel_h_shufC] + vinserti32x4 m16, [srcq+strideq*2], 2 lea srcq, [srcq+strideq*4] - vinserti128 ym1, [srcq+strideq*0], 1 - vinserti128 m1, [srcq+strideq*1], 2 - vinserti128 m1, [srcq+strideq*2], 3 + vinserti128 ym17, [srcq+strideq*0], 1 + vbroadcasti32x4 m20, [subpel_h_shufB] + vinserti32x4 m17, [srcq+strideq*1], 2 + vinserti32x4 m17, [srcq+strideq*2], 3 + pshufb m3, m16, m19 ; 0 1 2 3 0123 mova m2, m8 - mova m4, m8 + pshufb m0, m16, m21 ; 0 1 2 3 89ab + vpdpbusd m2, m3, m10 mova m3, m8 - mova m5, m8 - pshufb m20, m0, m16 - pshufb m21, m0, m17 - pshufb m22, m0, m18 - pshufb m23, m1, m16 - pshufb m6, m1, m17 - pshufb m7, m1, m18 - vpdpbusd m2, m20, m10 - vpdpbusd m4, m21, m10 - vpdpbusd m2, m21, m11 - vpdpbusd m4, m22, m11 - vpdpbusd m3, m23, m10 - vpdpbusd m5, m6, m10 - vpdpbusd m3, m6, m11 - vpdpbusd m5, m7, m11 - packssdw m2, m4 - packssdw m3, m5 - psraw m2, 2 ; _ 0 1 2 - psraw m3, 2 ; 3 4 5 6 - valignq m0, m3, m2, 2 ; 0 1 2 3 - valignq m1, m3, m2, 4 ; 1 2 3 4 - valignq m2, m3, m2, 6 ; 2 3 4 5 - punpcklwd m4, m0, m1 ; 01a 12a 23a 34a - punpckhwd m5, m0, m1 ; 01b 12b 23b 34b - punpcklwd m6, m2, m3 ; 23a 34a 45a 56a - punpckhwd m7, m2, m3 ; 23b 34b 45b 56b + pshufb m1, m17, m19 ; 3 4 5 6 0123 + vpdpbusd m3, m0, m11 + mova m0, m8 + pshufb m4, m17, m21 ; 3 4 5 6 89ab + vpdpbusd m0, m1, m10 + mova m1, m8 + pshufb m16, m20 ; 0 1 2 3 4567 + vpdpbusd m1, m4, m11 + pshufb m17, m20 ; 3 4 5 6 4567 + vpdpbusd m2, m16, m11 + vpdpbusd m3, m16, m10 + vpdpbusd m0, m17, m11 + vpdpbusd m1, m17, m10 + packssdw m2, m3 + packssdw m0, m1 + psraw m2, 2 ; 0 1 2 3 + psraw m0, 2 ; 3 4 5 6 + vshufi32x4 m4, m2, m0, q2132 ; 2 3 4 5 + vshufi32x4 m5, m2, m0, q1021 ; 1 2 3 4 + punpcklwd m3, m4, m0 ; 23 34 45 56 + punpckhwd m4, m0 + punpcklwd m1, m2, m5 ; 01 12 23 34 + punpckhwd m2, m5 .hv_w8_loop: - movu xm19, [srcq+stride3q ] + movu xm18, [srcq+stride3q ] lea srcq, [srcq+strideq*4] - vinserti128 ym19, [srcq+strideq*0], 1 - vinserti128 m19, [srcq+strideq*1], 2 - vinserti128 m19, [srcq+strideq*2], 3 - mova m20, m9 - mova m21, m9 - mova m22, m8 - mova m23, m8 - vpdpwssd m20, m4, m12 - vpdpwssd m21, m5, m12 - vpdpwssd m20, m6, m13 - vpdpwssd m21, m7, m13 - pshufb m0, m19, m16 - pshufb m1, m19, m17 - pshufb m2, m19, m18 - vpdpbusd m22, m0, m10 - vpdpbusd m23, m1, m10 - vpdpbusd m22, m1, m11 - vpdpbusd m23, m2, m11 - packssdw m22, m23 - psraw m22, 2 ; 7 8 9 A - valignq m0, m22, m3, 2 ; 4 5 6 7 - valignq m1, m22, m3, 4 ; 5 6 7 8 - valignq m2, m22, m3, 6 ; 6 7 8 9 - mova m3, m22 - punpcklwd m4, m0, m1 ; 45a 56a 67a 78a - punpckhwd m5, m0, m1 ; 45b 56b 67b 78b - punpcklwd m6, m2, m3 ; 67a 78a 89a 9Aa - punpckhwd m7, m2, m3 ; 67b 78b 89b 9Ab - vpdpwssd m20, m4, m14 - vpdpwssd m21, m5, m14 - vpdpwssd m20, m6, m15 - vpdpwssd m21, m7, m15 - psrad m20, 6 - psrad m21, 6 - packssdw m20, m21 - mova [tmpq], m20 + vinserti128 ym18, [srcq+strideq*0], 1 + vinserti32x4 m18, [srcq+strideq*1], 2 + vinserti32x4 m18, [srcq+strideq*2], 3 + pshufb m17, m18, m19 ; 7 8 9 a 0123 + mova m16, m8 + pshufb m5, m18, m21 ; 7 8 9 a 89ab + vpdpbusd m16, m17, m10 + mova m17, m8 + pshufb m18, m20 ; 7 8 9 a 4567 + vpdpbusd m17, m5, m11 + mova m5, m9 + vpdpwssd m5, m3, m13 ; a1 b1 c1 d1 + mova m6, m9 + vpdpwssd m6, m4, m13 + vpdpbusd m16, m18, m11 + vpdpbusd m17, m18, m10 + vpdpwssd m5, m1, m12 ; a0 b0 c0 d0 + mova m1, m3 + vpdpwssd m6, m2, m12 + mova m2, m4 + packssdw m16, m17 + psraw m16, 2 ; 7 8 9 a + valignq m4, m16, m0, 6 ; 6 7 8 9 + mova m0, m16 + punpcklwd m3, m4, m16 ; 67 78 89 9a + punpckhwd m4, m16 + vpdpwssd m5, m3, m15 ; a3 b3 c3 d3 + vpdpwssd m6, m4, m15 + vshufi32x4 m1, m3, q1032 ; 45 56 67 78 + vshufi32x4 m2, m4, q1032 + vpdpwssd m5, m1, m14 ; a2 b2 c2 d2 + vpdpwssd m6, m2, m14 + psrad m5, 6 + psrad m6, 6 + packssdw m5, m6 + mova [tmpq], m5 add tmpq, 64 sub hd, 4 jg .hv_w8_loop + vzeroupper RET .hv_w16: - mov wd, 16*2 - jmp .hv_start -.hv_w32: - mov wd, 32*2 - jmp .hv_start -.hv_w64: - mov wd, 64*2 - jmp .hv_start -.hv_w128: - mov wd, 128*2 -.hv_start: - WIN64_SPILL_XMM 31 - mova m16, [spel_h_perm16a] - mova m17, [spel_h_perm16b] - mova m18, [spel_h_perm16c] + WIN64_SPILL_XMM 23 + mova m16, [spel_h_perm16] + vpbroadcastd m18, [pb_4] + add wd, wd + paddb m17, m18, m16 lea r6d, [hq+wq*8-256] - mov r5, srcq + paddb m18, m17 +.hv_w16_loop0: + movu ym19, [srcq+strideq*0] + vinserti32x8 m19, [srcq+strideq*1], 1 + lea r5, [srcq+strideq*2] + movu ym20, [r5 +strideq*0] + vinserti32x8 m20, [r5 +strideq*1], 1 + lea r5, [r5 +strideq*2] + movu ym21, [r5 +strideq*0] + vinserti32x8 m21, [r5 +strideq*1], 1 + lea r5, [r5 +strideq*2] + movu ym22, [r5 +strideq*0] mov r7, tmpq -.hv_loop0: - movu ym0, [srcq+strideq*0] - vinserti32x8 m0, [srcq+strideq*1], 1 - lea srcq, [srcq+strideq*2] - movu ym1, [srcq+strideq*0] - vinserti32x8 m1, [srcq+strideq*1], 1 - lea srcq, [srcq+strideq*2] - movu ym2, [srcq+strideq*0] - vinserti32x8 m2, [srcq+strideq*1], 1 - lea srcq, [srcq+strideq*2] - movu ym3, [srcq+strideq*0] + vpermb m3, m16, m19 ; 0 1 0123 89ab + mova m2, m8 + vpermb m4, m18, m19 ; 0 1 89ab ghij + vpdpbusd m2, m3, m10 + mova m3, m8 + vpermb m5, m16, m20 ; 2 3 0123 89ab + vpdpbusd m3, m4, m11 mova m4, m8 + vpermb m6, m18, m20 ; 2 3 89ab ghij + vpdpbusd m4, m5, m10 mova m5, m8 + vpermb m7, m16, m21 ; 4 5 0123 89ab + vpdpbusd m5, m6, m11 mova m6, m8 + vpermb m0, m18, m21 ; 4 5 89ab ghij + vpdpbusd m6, m7, m10 mova m7, m8 - vpermb m19, m16, m0 - vpermb m20, m17, m0 - vpermb m21, m18, m0 - vpermb m22, m16, m1 - vpermb m23, m17, m1 - vpermb m24, m18, m1 - vpermb m25, m16, m2 - vpermb m26, m17, m2 - vpermb m27, m18, m2 - vpermb ym28, ym16, ym3 - vpermb ym29, ym17, ym3 - vpermb ym30, ym18, ym3 - mova m0, m8 - mova m1, m8 - mova ym2, ym8 - mova ym3, ym8 - vpdpbusd m4, m19, m10 - vpdpbusd m5, m20, m10 - vpdpbusd m6, m22, m10 - vpdpbusd m7, m23, m10 - vpdpbusd m0, m25, m10 - vpdpbusd m1, m26, m10 - vpdpbusd ym2, ym28, ym10 - vpdpbusd ym3, ym29, ym10 + vpermb ym1, ym16, ym22 ; 6 0123 89ab + vpdpbusd m7, m0, m11 + mova ym0, ym8 + vpermb m19, m17, m19 ; 0 1 4567 cdef + vpdpbusd ym0, ym1, ym10 + vpermb ym1, ym18, ym22 ; 6 89ab ghij + vpdpbusd m2, m19, m11 + vpdpbusd m3, m19, m10 + mova ym19, ym8 + vpermb m20, m17, m20 ; 2 3 4567 cdef + vpdpbusd ym19, ym1, ym11 + vpermb m21, m17, m21 ; 4 5 4567 cdef vpdpbusd m4, m20, m11 - vpdpbusd m5, m21, m11 - vpdpbusd m6, m23, m11 - vpdpbusd m7, m24, m11 - vpdpbusd m0, m26, m11 - vpdpbusd m1, m27, m11 - vpdpbusd ym2, ym29, ym11 - vpdpbusd ym3, ym30, ym11 - packssdw m4, m5 - packssdw m6, m7 - packssdw m0, m1 - packssdw ym2, ym3 - psraw m4, 2 ; 0a 0b 1a 1b - psraw m6, 2 ; 2a 2b 3a 3b - psraw m0, 2 ; 4a 4b 5a 5b - psraw ym2, 2 ; 6a 6b __ __ - vshufi32x4 m5, m4, m6, q1032 ; 1a 1b 2a 2b - vshufi32x4 m7, m6, m0, q1032 ; 3a 3b 4a 4b - vshufi32x4 m1, m0, m2, q1032 ; 5a 5b 6a 6b - punpcklwd m2, m4, m5 ; 01a 01c 12a 12c - punpckhwd m3, m4, m5 ; 01b 01d 12b 12d - punpcklwd m4, m6, m7 ; 23a 23c 34a 34c - punpckhwd m5, m6, m7 ; 23b 23d 34b 34d - punpcklwd m6, m0, m1 ; 45a 45c 56a 56c - punpckhwd m7, m0, m1 ; 45b 45d 56b 56d -.hv_loop: - movu ym19, [srcq+strideq*1] - lea srcq, [srcq+strideq*2] - vinserti32x8 m19, [srcq+strideq*0], 1 + vpdpbusd m5, m20, m10 + vpermb ym22, ym17, ym22 ; 6 4567 cdef + vpdpbusd m6, m21, m11 + vpdpbusd m7, m21, m10 + packssdw m2, m3 ; 0 1 + vpdpbusd ym0, ym22, ym11 + packssdw m4, m5 ; 2 3 + vpdpbusd ym19, ym22, ym10 + packssdw m6, m7 ; 4 5 + packssdw ym0, ym19 ; 6 + REPX {psraw x, 2}, m2, m4, m6, ym0 + vshufi32x4 m3, m2, m4, q1032 ; 1 2 + vshufi32x4 m5, m4, m6, q1032 ; 3 4 + vshufi32x4 m0, m6, m0, q1032 ; 5 6 + punpcklwd m1, m2, m3 ; 01 12 + punpckhwd m2, m3 + punpcklwd m3, m4, m5 ; 23 34 + punpckhwd m4, m5 + punpcklwd m5, m6, m0 ; 45 56 + punpckhwd m6, m0 +.hv_w16_loop: + movu ym19, [r5+strideq*1] + lea r5, [r5+strideq*2] + vinserti32x8 m19, [r5+strideq*0], 1 mova m20, m9 + vpdpwssd m20, m1, m12 ; a0 + vpermb m1, m16, m19 mova m21, m9 + vpdpwssd m21, m2, m12 ; b0 + vpermb m2, m17, m19 mova m22, m8 - mova m23, m8 - vpdpwssd m20, m2, m12 - vpdpwssd m21, m3, m12 - vpdpwssd m20, m4, m13 - vpdpwssd m21, m5, m13 - vpermb m24, m16, m19 - vpermb m25, m17, m19 - vpermb m26, m18, m19 - vpdpbusd m22, m24, m10 - vpdpbusd m23, m25, m10 - vpdpbusd m22, m25, m11 - vpdpbusd m23, m26, m11 - packssdw m22, m23 - psraw m22, 2 ; 7a 7b 8a 8b - vshufi32x4 m0, m1, m22, q1032 ; 6a 6b 7a 7b + vpdpbusd m22, m1, m10 + mova m1, m8 + vpermb m19, m18, m19 + vpdpbusd m1, m2, m10 + vpdpwssd m20, m3, m13 ; a1 + vpdpwssd m21, m4, m13 ; b1 + vpdpbusd m22, m2, m11 mova m2, m4 - mova m3, m5 - mova m1, m22 + vpdpbusd m1, m19, m11 mova m4, m6 - mova m5, m7 - punpcklwd m6, m0, m1 ; 67a 67c 78a 78c - punpckhwd m7, m0, m1 ; 67b 67d 78b 78d - vpdpwssd m20, m4, m14 - vpdpwssd m21, m5, m14 - vpdpwssd m20, m6, m15 - vpdpwssd m21, m7, m15 + vpdpwssd m20, m5, m14 ; a2 + vpdpwssd m21, m6, m14 ; b2 + packssdw m22, m1 + mova m1, m3 + psraw m22, 2 ; 7 8 + mova m3, m5 + vshufi32x4 m6, m0, m22, q1032 ; 6 7 + mova m0, m22 + punpcklwd m5, m6, m0 ; 67 78 + punpckhwd m6, m0 + vpdpwssd m20, m5, m15 ; a3 + vpdpwssd m21, m6, m15 ; b3 psrad m20, 6 psrad m21, 6 packssdw m20, m21 - mova [tmpq+wq*0], ym20 - vextracti32x8 [tmpq+wq*1], m20, 1 - lea tmpq, [tmpq+wq*2] + mova [r7+wq*0], ym20 + vextracti32x8 [r7+wq*1], m20, 1 + lea r7, [r7+wq*2] sub hd, 2 - jg .hv_loop - add r5, 16 - add r7, 32 + jg .hv_w16_loop + add srcq, 16 + add tmpq, 32 movzx hd, r6b - mov srcq, r5 - mov tmpq, r7 sub r6d, 1<<8 - jg .hv_loop0 + jg .hv_w16_loop0 RET cglobal warp_affine_8x8t_8bpc, 4, 7, 22, tmp, ts |